Browse Source

Add optional header and status line length caps

master
Jens Pitkänen 4 months ago
parent
commit
efbaf75751
10 changed files with 196 additions and 28 deletions
  1. +6
    -0
      CHANGELOG.md
  2. +2
    -2
      Cargo.toml
  3. +4
    -0
      README.md
  4. +15
    -5
      src/connection.rs
  5. +12
    -0
      src/error.rs
  6. +1
    -1
      src/lib.rs
  7. +46
    -0
      src/request.rs
  8. +55
    -13
      src/response.rs
  9. +38
    -0
      tests/main.rs
  10. +17
    -7
      tests/setup.rs

+ 6
- 0
CHANGELOG.md View File

@ -9,11 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `Request::with_param` for more ergonomic query parameter
usage. Thanks for the PR, @sjvignesh!
([#54](https://github.com/neonmoe/minreq/pull/54))
- `Request::with_max_headers_size` and
`Request::with_max_status_line_length` for avoiding DoS when the
server sends large headers or status lines. Thanks for the report,
@Shnatsel! ([#55](https://github.com/neonmoe/minreq/issues/55))
### Fixed
- Chunk length handling for some servers with slightly off-spec chunk
lengths. Thanks for the report, @Shnatsel!
([#50](https://github.com/neonmoe/minreq/issues/50))
- Timeouts not always being properly enforced. Thanks for the report,
@Shnatsel! ([#52](https://github.com/neonmoe/minreq/issues/52))
## [2.3.1] - 2021-02-10
### Fixed


+ 2
- 2
Cargo.toml View File

@ -1,6 +1,6 @@
[package]
name = "minreq"
version = "2.3.2-alpha.0"
version = "2.4.0-alpha.0"
authors = ["Jens Pitkanen <jens@neon.moe>"]
description = "Simple, minimal-dependency HTTP client"
documentation = "https://docs.rs/minreq"
@ -35,7 +35,7 @@ openssl-probe = { version = "0.1", optional = true }
native-tls = { version = "0.2", optional = true }
[dev-dependencies]
tiny_http = "^0.6.2"
tiny_http = "^0.8.2"
serde_derive = "^1.0.101"
env_logger = "^0.8.3"


+ 4
- 0
README.md View File

@ -30,6 +30,10 @@ major version bump.
- Change the response/request structs to allow multiple headers with
the same name.
- Set sane defaults for maximum header size and status line
length. The ability to add maximums was added in response to
[#55](https://github.com/neonmoe/minreq/issues/55), but defaults for
the limits is a breaking change.
## License
This crate is distributed under the terms of the [ISC license](COPYING.md).

+ 15
- 5
src/connection.rs View File

@ -157,8 +157,11 @@ impl Connection {
// Receive request
log::trace!("Reading HTTPS response from {}.", self.request.host);
let response =
ResponseLazy::from_stream(HttpStream::create_secured(tls, self.timeout_at))?;
let response = ResponseLazy::from_stream(
HttpStream::create_secured(tls, self.timeout_at),
self.request.max_headers_size,
self.request.max_status_line_len,
)?;
handle_redirects(self, response)
})
}
@ -201,8 +204,11 @@ impl Connection {
// Receive request
log::trace!("Reading HTTPS response from {}.", self.request.host);
let response =
ResponseLazy::from_stream(HttpStream::create_secured(tls, self.timeout_at))?;
let response = ResponseLazy::from_stream(
HttpStream::create_secured(tls, self.timeout_at),
self.request.max_headers_size,
self.request.max_status_line_len,
)?;
handle_redirects(self, response)
})
}
@ -234,7 +240,11 @@ impl Connection {
}
};
let stream = HttpStream::create_unsecured(BufReader::new(tcp), self.timeout_at);
let response = ResponseLazy::from_stream(stream)?;
let response = ResponseLazy::from_stream(
stream,
self.request.max_headers_size,
self.request.max_status_line_len,
)?;
handle_redirects(self, response)
})
}


+ 12
- 0
src/error.rs View File

@ -15,9 +15,18 @@ pub enum Error {
/// Couldn't parse the incoming chunk's length while receiving a
/// response with the header `Transfer-Encoding: chunked`.
MalformedChunkLength,
/// The chunk did not end after reading the previously read amount
/// of bytes.
MalformedChunkEnd,
/// Couldn't parse the `Content-Length` header's value as an
/// `usize`.
MalformedContentLength,
/// The response contains headers whose total size surpasses
/// [Request::with_max_headers_size](crate::request::Request::with_max_headers_size).
HeadersOverflow,
/// The response's status line length surpasses
/// [Request::with_max_status_line_size](crate::request::Request::with_max_status_line_length).
StatusLineOverflow,
/// [ToSocketAddrs](std::net::ToSocketAddrs) did not resolve to an
/// address.
AddressNotFound,
@ -73,7 +82,10 @@ impl fmt::Display for Error {
InvalidUtf8InBody(err) => write!(f, "{}", err),
MalformedChunkLength => write!(f, "non-usize chunk length with transfer-encoding: chunked"),
MalformedChunkEnd => write!(f, "chunk did not end after reading the expected amount of bytes"),
MalformedContentLength => write!(f, "non-usize content length"),
HeadersOverflow => write!(f, "the headers' total size surpassed max_headers_size"),
StatusLineOverflow => write!(f, "the status line length surpassed max_status_line_length"),
AddressNotFound => write!(f, "could not resolve host to a socket address"),
RedirectLocationMissing => write!(f, "redirection location header missing"),
InfiniteRedirectionLoop => write!(f, "infinite redirection loop detected"),


+ 1
- 1
src/lib.rs View File

@ -21,7 +21,7 @@
//!
//! ```toml
//! [dependencies]
//! minreq = { version = "2.3.2-alpha.0", features = ["punycode"] }
//! minreq = { version = "2.4.0-alpha.0", features = ["punycode"] }
//! ```
//!
//! Below is the list of all available features.


+ 46
- 0
src/request.rs View File

@ -76,6 +76,8 @@ pub struct Request {
headers: HashMap<String, String>,
body: Option<Vec<u8>>,
pub(crate) timeout: Option<u64>,
pub(crate) max_headers_size: Option<usize>,
pub(crate) max_status_line_len: Option<usize>,
max_redirects: usize,
pub(crate) https: bool,
pub(crate) redirects: Vec<(bool, URL, URL)>,
@ -97,6 +99,8 @@ impl Request {
headers: HashMap::new(),
body: None,
timeout: None,
max_headers_size: None,
max_status_line_len: None,
max_redirects: 100,
https,
redirects: Vec::new(),
@ -174,6 +178,48 @@ impl Request {
self
}
/// Sets the maximum size of all the headers this request will
/// accept.
///
/// If this limit is passed, the request will close the connection
/// and return an [Error::HeadersOverflow] error.
///
/// The maximum length is counted in bytes, including line-endings
/// and other whitespace. Both normal and trailing headers count
/// towards this cap.
///
/// `None` disables the cap, and may cause the program to use any
/// amount of memory if the server responds with a lot of headers
/// (or an infinite amount). In minreq versions 2.x.x, the default
/// is None, so setting this manually is recommended when talking
/// to untrusted servers.
pub fn with_max_headers_size<S: Into<Option<usize>>>(mut self, max_headers_size: S) -> Request {
self.max_headers_size = max_headers_size.into();
self
}
/// Sets the maximum length of the status line this request will
/// accept.
///
/// If this limit is passed, the request will close the connection
/// and return an [Error::StatusLineOverflow] error.
///
/// The maximum length is counted in bytes, including the
/// line-ending `\r\n`.
///
/// `None` disables the cap, and may cause the program to use any
/// amount of memory if the server responds with a long (or
/// infinite) status line. In minreq versions 2.x.x, the default
/// is None, so setting this manually is recommended when talking
/// to untrusted servers.
pub fn with_max_status_line_length<S: Into<Option<usize>>>(
mut self,
max_status_line_len: S,
) -> Request {
self.max_status_line_len = max_status_line_len.into();
self
}
/// Sends this request to the host.
///
/// # Errors


+ 55
- 13
src/response.rs View File

@ -210,17 +210,23 @@ pub struct ResponseLazy {
stream: Bytes<HttpStream>,
state: HttpStreamState,
max_trailing_headers_size: Option<usize>,
}
impl ResponseLazy {
pub(crate) fn from_stream(stream: HttpStream) -> Result<ResponseLazy, Error> {
pub(crate) fn from_stream(
stream: HttpStream,
max_headers_size: Option<usize>,
max_status_line_len: Option<usize>,
) -> Result<ResponseLazy, Error> {
let mut stream = stream.bytes();
let ResponseMetadata {
status_code,
reason_phrase,
headers,
state,
} = read_metadata(&mut stream)?;
max_trailing_headers_size,
} = read_metadata(&mut stream, max_headers_size, max_status_line_len)?;
Ok(ResponseLazy {
status_code,
@ -228,6 +234,7 @@ impl ResponseLazy {
headers,
stream,
state,
max_trailing_headers_size,
})
}
}
@ -247,6 +254,7 @@ impl Iterator for ResponseLazy {
expecting_chunks,
length,
content_length,
self.max_trailing_headers_size,
)
}
}
@ -284,9 +292,13 @@ fn read_with_content_length(
fn read_trailers(
bytes: &mut Bytes<HttpStream>,
headers: &mut HashMap<String, String>,
mut max_headers_size: Option<usize>,
) -> Result<(), Error> {
loop {
let trailer_line = read_line(bytes)?;
let trailer_line = read_line(bytes, max_headers_size, Error::HeadersOverflow)?;
if let Some(ref mut max_headers_size) = max_headers_size {
*max_headers_size -= trailer_line.len() + 2;
}
if let Some((header, value)) = parse_header(trailer_line) {
headers.insert(header, value);
} else {
@ -302,14 +314,19 @@ fn read_chunked(
expecting_more_chunks: &mut bool,
chunk_length: &mut usize,
content_length: &mut usize,
max_trailing_headers_size: Option<usize>,
) -> Option<<ResponseLazy as Iterator>::Item> {
if !*expecting_more_chunks && *chunk_length == 0 {
return None;
}
if *chunk_length == 0 {
// Max length of the chunk length line is 1KB: not too long to
// take up much memory, long enough to tolerate some chunk
// extensions (which are ignored).
// Get the size of the next chunk
let length_line = match read_line(bytes) {
let length_line = match read_line(bytes, Some(1024), Error::MalformedChunkLength) {
Ok(line) => line,
Err(err) => return Some(Err(err)),
};
@ -320,14 +337,19 @@ fn read_chunked(
let incoming_length = if length_line.is_empty() {
0
} else {
match usize::from_str_radix(length_line.trim(), 16) {
let length = if let Some(i) = length_line.find(";") {
length_line[..i].trim()
} else {
length_line.trim()
};
match usize::from_str_radix(length, 16) {
Ok(length) => length,
Err(_) => return Some(Err(Error::MalformedChunkLength)),
}
};
if incoming_length == 0 {
if let Err(err) = read_trailers(bytes, headers) {
if let Err(err) = read_trailers(bytes, headers, max_trailing_headers_size) {
return Some(Err(err));
}
@ -353,7 +375,7 @@ fn read_chunked(
// TODO: Maybe this could be written in a way
// that doesn't discard the last ok byte if
// the \r\n reading fails?
if let Err(err) = read_line(bytes) {
if let Err(err) = read_line(bytes, Some(2), Error::MalformedChunkEnd) {
return Some(Err(err));
}
}
@ -392,18 +414,27 @@ struct ResponseMetadata {
reason_phrase: String,
headers: HashMap<String, String>,
state: HttpStreamState,
max_trailing_headers_size: Option<usize>,
}
fn read_metadata(stream: &mut Bytes<HttpStream>) -> Result<ResponseMetadata, Error> {
let (status_code, reason_phrase) = parse_status_line(&read_line(stream)?);
fn read_metadata(
stream: &mut Bytes<HttpStream>,
mut max_headers_size: Option<usize>,
max_status_line_len: Option<usize>,
) -> Result<ResponseMetadata, Error> {
let line = read_line(stream, max_status_line_len, Error::StatusLineOverflow)?;
let (status_code, reason_phrase) = parse_status_line(&line);
let mut headers = HashMap::new();
loop {
let line = read_line(stream)?;
let line = read_line(stream, max_headers_size, Error::HeadersOverflow)?;
if line.is_empty() {
// Body starts here
break;
}
if let Some(ref mut max_headers_size) = max_headers_size {
*max_headers_size -= line.len() + 2;
}
if let Some(header) = parse_header(line) {
headers.insert(header.0, header.1);
}
@ -441,16 +472,27 @@ fn read_metadata(stream: &mut Bytes<HttpStream>) -> Result<ResponseMetadata, Err
reason_phrase,
headers,
state,
max_trailing_headers_size: max_headers_size,
})
}
fn read_line(stream: &mut Bytes<HttpStream>) -> Result<String, Error> {
fn read_line(
stream: &mut Bytes<HttpStream>,
max_len: Option<usize>,
overflow_error: Error,
) -> Result<String, Error> {
let mut bytes = Vec::with_capacity(32);
for byte in stream {
let byte = byte?;
if let Some(max_len) = max_len {
if bytes.len() >= max_len {
return Err(overflow_error);
}
}
if byte == b'\n' {
// Pop the \r off, as HTTP lines end in \r\n.
bytes.pop();
if let Some(b'\r') = bytes.last() {
bytes.pop();
}
break;
} else {
bytes.push(byte);


+ 38
- 0
tests/main.rs View File

@ -225,3 +225,41 @@ fn tcp_connect_timeout() {
)
);
}
#[test]
fn test_header_cap() {
setup();
let body = minreq::get(url("/long_header"))
.with_max_headers_size(999)
.send();
assert!(body.is_err());
assert_eq!(
format!("{:?}", body.err().unwrap()),
format!("{:?}", minreq::Error::HeadersOverflow)
);
let body = minreq::get(url("/long_header"))
.with_max_headers_size(1500)
.send();
assert!(body.is_ok());
}
#[test]
fn test_status_line_cap() {
setup();
let expected_status_line = "HTTP/1.1 203 Non-Authoritative Information";
let body = minreq::get(url("/long_status_line"))
.with_max_status_line_length(expected_status_line.len() + 1)
.send();
assert!(body.is_err());
assert_eq!(
format!("{:?}", body.err().unwrap()),
format!("{:?}", minreq::Error::StatusLineOverflow)
);
let body = minreq::get(url("/long_status_line"))
.with_max_status_line_length(expected_status_line.len() + 2)
.send();
assert!(body.is_ok());
}

+ 17
- 7
tests/setup.rs View File

@ -1,8 +1,8 @@
extern crate minreq;
extern crate tiny_http;
use self::tiny_http::{Header, Method, Response, Server};
use std::sync::Arc;
use std::sync::Once;
use std::str::FromStr;
use std::sync::{Arc, Once};
use std::thread;
use std::time::Duration;
@ -55,13 +55,23 @@ pub fn setup() {
request.respond(response).ok();
}
Method::Get if url == "/long_header" => {
let mut long_header = String::with_capacity(1000);
long_header += "Very-Long-Header: ";
for _ in 0..1000 - long_header.len() {
long_header += ".";
}
let long_header = Header::from_str(&long_header).unwrap();
let response = Response::empty(200).with_header(long_header);
request.respond(response).ok();
}
Method::Get if url == "/long_status_line" => {
request.respond(Response::empty(203)).ok();
}
Method::Get if url == "/redirect-baz" => {
let response = Response::empty(301).with_header(
Header::from_bytes(
&b"Location"[..],
&b"http://localhost:35562/a#baz"[..],
)
.unwrap(),
Header::from_str("Location: http://localhost:35562/a#baz").unwrap(),
);
request.respond(response).ok();
}


Loading…
Cancel
Save