From b1cf1f9eb43fbe4976b4063f47a0a2c259f535a9 Mon Sep 17 00:00:00 2001 From: Melody Horn Date: Sat, 27 Mar 2021 10:30:49 -0600 Subject: specify user agent --- src/download.rs | 19 +++++++++++++------ src/main.rs | 4 ++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/download.rs b/src/download.rs index fa99ea5..c59eec3 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,10 +1,10 @@ use std::path::{Path, PathBuf}; -use anyhow::{bail, Error, Context}; -use hyper::Uri; +use anyhow::{bail, Context, Error, Result}; +use hyper::{Body, Request, Uri}; use hyper::body::HttpBody; use hyper::client::{Client, HttpConnector}; -use hyper::header::{CONTENT_TYPE, LOCATION}; +use hyper::header::{CONTENT_TYPE, LOCATION, USER_AGENT}; use hyper_rustls::HttpsConnector; use tokio::fs; use tokio::io::AsyncWriteExt; @@ -13,7 +13,14 @@ use super::Args; type HttpsClient = Client>; -pub(crate) async fn download(url: Uri, args: &Args) -> Result<(), Error> { +fn request(url: Uri, args: &Args) -> Result> { + Request::get(url) + .header(USER_AGENT, args.user_agent.clone()) + .body(Body::empty()) + .map_err(Error::new) +} + +pub(crate) async fn download(url: Uri, args: &Args) -> Result<()> { let output_file_dir = &args.directory_prefix; let output_file_path = if let Some(output) = &args.output_document { @@ -27,13 +34,13 @@ pub(crate) async fn download(url: Uri, args: &Args) -> Result<(), Error> { }; let client: HttpsClient = Client::builder().build(HttpsConnector::with_native_roots()); - let mut response = client.get(url.clone()).await?; + let mut response = client.request(request(url.clone(), args)?).await?; while response.status().is_redirection() { let location = response.headers() .get(LOCATION).context("no Location header in redirect")? .to_str().context("malformed Location header in redirect")? .parse().context("non-URL Location header in redirect")?; - response = client.get(location).await?; + response = client.request(request(location, args)?).await?; } if !response.status().is_success() { let status = response.status(); diff --git a/src/main.rs b/src/main.rs index a622afd..5de67ca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,6 +24,10 @@ struct Args { #[structopt(short = "P", long, default_value = ".")] directory_prefix: PathBuf, + /// User-Agent header to include + #[structopt(short = "U", long, default_value = concat!("webget/", env!("CARGO_PKG_VERSION")))] + user_agent: String, + /// The URLs to download urls: Vec, } -- cgit v1.2.3