1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
|
use std::path::{Path, PathBuf};
use anyhow::{bail, Context, Error, Result};
use hyper::{Body, Request, Uri};
use hyper::body::HttpBody;
use hyper::client::{Client, HttpConnector};
use hyper::header::{CONTENT_TYPE, LOCATION, USER_AGENT};
use hyper_rustls::HttpsConnector;
use tokio::fs;
use tokio::io::AsyncWriteExt;
use super::Args;
type HttpsClient = Client<HttpsConnector<HttpConnector>>;
fn request(url: Uri, args: &Args) -> Result<Request<Body>> {
Request::get(url)
.header(USER_AGENT, args.user_agent.clone())
.body(Body::empty())
.map_err(Error::new)
}
pub(crate) async fn download(url: Uri, args: &Args) -> Result<()> {
let output_file_dir = &args.directory_prefix;
let output_file_path = if let Some(output) = &args.output_document {
Some(output.clone())
} else {
let url_path = Path::new(url.path());
match (url_path.file_name(), url.path().ends_with('/')) {
(Some(file_name), false) => Some(PathBuf::from(file_name)),
_ => None,
}
};
let client: HttpsClient = Client::builder().build(HttpsConnector::with_native_roots());
let mut response = client.request(request(url.clone(), args)?).await?;
while response.status().is_redirection() {
let location = response.headers()
.get(LOCATION).context("no Location header in redirect")?
.to_str().context("malformed Location header in redirect")?
.parse().context("non-URL Location header in redirect")?;
response = client.request(request(location, args)?).await?;
}
if !response.status().is_success() {
let status = response.status();
let body = hyper::body::to_bytes(response.into_body()).await?;
let body = String::from_utf8_lossy(&body);
bail!("non-success response code {} in URL {}:\n{}\n", status, url, body);
}
let output_file_path = if let Some(path) = output_file_path {
path
} else {
let content_type = response.headers().get(CONTENT_TYPE);
let extension = content_type
.and_then(|mime_type| mime_type.to_str().ok())
.and_then(|mime_type| mime2ext::mime2ext(mime_type))
.map(|x| format!(".{}", x))
.unwrap_or_default();
PathBuf::from(format!("index{}", extension))
};
let output_file_path = output_file_dir.join(output_file_path);
let mut output_file = fs::File::create(output_file_path).await?;
while let Some(data) = response.body_mut().data().await {
output_file.write_all(&data?).await?;
}
Ok(())
}
|
