use anyhow::{anyhow, bail, Context as _, Result};
use bytes::Bytes;
use http_body_util::BodyExt;
use hyper_util::rt::TokioIo;
use mime::Mime;
use serde::Serialize;
use crate::context::Context;
use crate::net::proxy::ProxyConfig;
use crate::net::session::SessionStream;
use crate::net::tls::wrap_rustls;
#[derive(Debug)]
pub struct Response {
pub blob: Vec<u8>,
pub mimetype: Option<String>,
pub encoding: Option<String>,
}
pub async fn read_url(context: &Context, url: &str) -> Result<String> {
let response = read_url_blob(context, url).await?;
let text = String::from_utf8_lossy(&response.blob);
Ok(text.to_string())
}
async fn get_http_sender<B>(
context: &Context,
parsed_url: hyper::Uri,
) -> Result<hyper::client::conn::http1::SendRequest<B>>
where
B: hyper::body::Body + 'static + Send,
B::Data: Send,
B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
{
let scheme = parsed_url.scheme_str().context("URL has no scheme")?;
let host = parsed_url.host().context("URL has no host")?;
let proxy_config_opt = ProxyConfig::load(context).await?;
let stream: Box<dyn SessionStream> = match scheme {
"http" => {
let port = parsed_url.port_u16().unwrap_or(80);
let load_cache = false;
if let Some(proxy_config) = proxy_config_opt {
let proxy_stream = proxy_config
.connect(context, host, port, load_cache)
.await?;
Box::new(proxy_stream)
} else {
let tcp_stream = crate::net::connect_tcp(context, host, port, load_cache).await?;
Box::new(tcp_stream)
}
}
"https" => {
let port = parsed_url.port_u16().unwrap_or(443);
let load_cache = true;
if let Some(proxy_config) = proxy_config_opt {
let proxy_stream = proxy_config
.connect(context, host, port, load_cache)
.await?;
let tls_stream = wrap_rustls(host, &[], proxy_stream).await?;
Box::new(tls_stream)
} else {
let tcp_stream = crate::net::connect_tcp(context, host, port, load_cache).await?;
let tls_stream = wrap_rustls(host, &[], tcp_stream).await?;
Box::new(tls_stream)
}
}
_ => bail!("Unknown URL scheme"),
};
let io = TokioIo::new(stream);
let (sender, conn) = hyper::client::conn::http1::handshake(io).await?;
tokio::task::spawn(conn);
Ok(sender)
}
pub async fn read_url_blob(context: &Context, url: &str) -> Result<Response> {
let mut url = url.to_string();
for _i in 0..10 {
let parsed_url = url
.parse::<hyper::Uri>()
.with_context(|| format!("Failed to parse URL {url:?}"))?;
let mut sender = get_http_sender(context, parsed_url.clone()).await?;
let authority = parsed_url
.authority()
.context("URL has no authority")?
.clone();
let req = hyper::Request::builder()
.uri(parsed_url.path())
.header(hyper::header::HOST, authority.as_str())
.body(http_body_util::Empty::<Bytes>::new())?;
let response = sender.send_request(req).await?;
if response.status().is_redirection() {
let header = response
.headers()
.get_all("location")
.iter()
.last()
.ok_or_else(|| anyhow!("Redirection doesn't have a target location"))?
.to_str()?;
info!(context, "Following redirect to {}", header);
url = header.to_string();
continue;
}
let content_type = response
.headers()
.get("content-type")
.and_then(|value| value.to_str().ok())
.and_then(|value| value.parse::<Mime>().ok());
let mimetype = content_type
.as_ref()
.map(|mime| mime.essence_str().to_string());
let encoding = content_type.as_ref().and_then(|mime| {
mime.get_param(mime::CHARSET)
.map(|charset| charset.as_str().to_string())
});
let body = response.collect().await?.to_bytes();
let blob: Vec<u8> = body.to_vec();
return Ok(Response {
blob,
mimetype,
encoding,
});
}
Err(anyhow!("Followed 10 redirections"))
}
pub(crate) async fn post_empty(context: &Context, url: &str) -> Result<(String, bool)> {
let parsed_url = url
.parse::<hyper::Uri>()
.with_context(|| format!("Failed to parse URL {url:?}"))?;
let scheme = parsed_url.scheme_str().context("URL has no scheme")?;
if scheme != "https" {
bail!("POST requests to non-HTTPS URLs are not allowed");
}
let mut sender = get_http_sender(context, parsed_url.clone()).await?;
let authority = parsed_url
.authority()
.context("URL has no authority")?
.clone();
let req = hyper::Request::post(parsed_url.path())
.header(hyper::header::HOST, authority.as_str())
.body(http_body_util::Empty::<Bytes>::new())?;
let response = sender.send_request(req).await?;
let response_status = response.status();
let body = response.collect().await?.to_bytes();
let text = String::from_utf8_lossy(&body);
let response_text = text.to_string();
Ok((response_text, response_status.is_success()))
}
#[allow(dead_code)]
pub(crate) async fn post_string(context: &Context, url: &str, body: String) -> Result<bool> {
let parsed_url = url
.parse::<hyper::Uri>()
.with_context(|| format!("Failed to parse URL {url:?}"))?;
let scheme = parsed_url.scheme_str().context("URL has no scheme")?;
if scheme != "https" {
bail!("POST requests to non-HTTPS URLs are not allowed");
}
let mut sender = get_http_sender(context, parsed_url.clone()).await?;
let authority = parsed_url
.authority()
.context("URL has no authority")?
.clone();
let request = hyper::Request::post(parsed_url.path())
.header(hyper::header::HOST, authority.as_str())
.body(body)?;
let response = sender.send_request(request).await?;
Ok(response.status().is_success())
}
pub(crate) async fn post_form<T: Serialize + ?Sized>(
context: &Context,
url: &str,
form: &T,
) -> Result<Bytes> {
let parsed_url = url
.parse::<hyper::Uri>()
.with_context(|| format!("Failed to parse URL {url:?}"))?;
let scheme = parsed_url.scheme_str().context("URL has no scheme")?;
if scheme != "https" {
bail!("POST requests to non-HTTPS URLs are not allowed");
}
let encoded_body = serde_urlencoded::to_string(form).context("Failed to encode data")?;
let mut sender = get_http_sender(context, parsed_url.clone()).await?;
let authority = parsed_url
.authority()
.context("URL has no authority")?
.clone();
let request = hyper::Request::post(parsed_url.path())
.header(hyper::header::HOST, authority.as_str())
.header("content-type", "application/x-www-form-urlencoded")
.body(encoded_body)?;
let response = sender.send_request(request).await?;
let bytes = response.collect().await?.to_bytes();
Ok(bytes)
}