use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Error, Result}; use hyper::{Body, Request, Uri}; use hyper::body::HttpBody; use hyper::client::{Client, HttpConnector}; use hyper::header::{CONTENT_TYPE, LOCATION, USER_AGENT}; use hyper_rustls::HttpsConnector; use tokio::fs; use tokio::io::AsyncWriteExt; use super::Args; type HttpsClient = Client>; fn request(url: Uri, args: &Args) -> Result> { Request::get(url) .header(USER_AGENT, args.user_agent.clone()) .body(Body::empty()) .map_err(Error::new) } pub(crate) async fn download(url: Uri, args: &Args) -> Result<()> { let output_file_dir = &args.directory_prefix; let output_file_path = if let Some(output) = &args.output_document { Some(output.clone()) } else { let url_path = Path::new(url.path()); match (url_path.file_name(), url.path().ends_with('/')) { (Some(file_name), false) => Some(PathBuf::from(file_name)), _ => None, } }; let client: HttpsClient = Client::builder().build(HttpsConnector::with_native_roots()); let mut response = client.request(request(url.clone(), args)?).await?; while response.status().is_redirection() { let location = response.headers() .get(LOCATION).context("no Location header in redirect")? .to_str().context("malformed Location header in redirect")? .parse().context("non-URL Location header in redirect")?; response = client.request(request(location, args)?).await?; } if !response.status().is_success() { let status = response.status(); let body = hyper::body::to_bytes(response.into_body()).await?; let body = String::from_utf8_lossy(&body); bail!("non-success response code {} in URL {}:\n{}\n", status, url, body); } let output_file_path = if let Some(path) = output_file_path { path } else { let content_type = response.headers().get(CONTENT_TYPE); let extension = content_type .and_then(|mime_type| mime_type.to_str().ok()) .and_then(|mime_type| mime2ext::mime2ext(mime_type)) .map(|x| format!(".{}", x)) .unwrap_or_default(); PathBuf::from(format!("index{}", extension)) }; let output_file_path = output_file_dir.join(output_file_path); let mut output_file = fs::File::create(output_file_path).await?; while let Some(data) = response.body_mut().data().await { output_file.write_all(&data?).await?; } Ok(()) }