From 285d606756b2de628b957ac8792ce8f0539ec6f6 Mon Sep 17 00:00:00 2001 From: "patchback[bot]" <45432694+patchback[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 10:34:37 -1000 Subject: [PATCH] [PR #12302/2dc02ee0 backport][3.13] Skip duplicate singleton header check in lax mode (#12303) Co-authored-by: J. Nick Koston Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Fixes home-assistant/core#166956 Fixes https://github.com/getmoto/moto/issues/9930 Fixes #12301 Fixes https://github.com/catalyst-cooperative/pudl-archiver/issues/1059 CVE: CVE-2026-34525 Upstream-Status: Backport [https://github.com/aio-libs/aiohttp/commit/53e2e6fc58b89c6185be7820bd2c9f40216b3000] (cherry picked from commit 53e2e6fc58b89c6185be7820bd2c9f40216b3000) Signed-off-by: Darsh Kelaiya --- CHANGES/12302.bugfix.rst | 3 ++ aiohttp/_http_parser.pyx | 27 ++++++----- aiohttp/http_parser.py | 40 ++++++++-------- tests/test_http_parser.py | 97 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 131 insertions(+), 36 deletions(-) create mode 100644 CHANGES/12302.bugfix.rst diff --git a/CHANGES/12302.bugfix.rst b/CHANGES/12302.bugfix.rst new file mode 100644 index 000000000..fe9e8fbd6 --- /dev/null +++ b/CHANGES/12302.bugfix.rst @@ -0,0 +1,3 @@ +Skipped the duplicate singleton header check in lax mode (the default for response +parsing). In strict mode (request parsing, or ``-X dev``), all RFC 9110 singletons +are still enforced -- by :user:`bdraco`. diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index bb7bf673d..8e9ecae69 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -72,8 +72,11 @@ cdef object StreamReader = _StreamReader cdef object DeflateBuffer = _DeflateBuffer -# https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-6 -cdef tuple SINGLETON_HEADERS = ( +# RFC 9110 singleton headers — duplicates are rejected in strict mode. +# In lax mode (response parser default), the check is skipped entirely +# since real-world servers (e.g. Google APIs, Werkzeug) commonly send +# duplicate headers like Content-Type or Server. +cdef frozenset SINGLETON_HEADERS = frozenset({ hdrs.CONTENT_LENGTH, hdrs.CONTENT_LOCATION, hdrs.CONTENT_RANGE, @@ -84,7 +87,7 @@ cdef tuple SINGLETON_HEADERS = ( hdrs.SERVER, hdrs.TRANSFER_ENCODING, hdrs.USER_AGENT, -) +}) cdef inline object extend(object buf, const char* at, size_t length): cdef Py_ssize_t s @@ -304,6 +307,7 @@ cdef class HttpParser: size_t _max_headers bint _response_with_body bint _read_until_eof + bint _lax bint _started object _url @@ -311,6 +315,7 @@ cdef class HttpParser: str _path str _reason object _headers + set _seen_singletons list _raw_headers bint _upgraded list _messages @@ -377,6 +382,8 @@ cdef class HttpParser: self._upgraded = False self._auto_decompress = auto_decompress self._content_encoding = None + self._lax = False + self._seen_singletons = set() self._csettings.on_url = cb_on_url self._csettings.on_status = cb_on_status @@ -407,6 +414,10 @@ cdef class HttpParser: if "\x00" in value: raise InvalidHeader(raw_value) + if not self._lax and name in SINGLETON_HEADERS: + if name in self._seen_singletons: + raise BadHttpMessage(f"Duplicate '{name}' header found.") + self._seen_singletons.add(name) self._headers.add(name, value) if len(self._headers) > self._max_headers: raise BadHttpMessage("Too many headers received") @@ -452,14 +463,6 @@ cdef class HttpParser: raw_headers = tuple(self._raw_headers) headers = CIMultiDictProxy(self._headers) - # https://www.rfc-editor.org/rfc/rfc9110.html#name-collected-abnf - bad_hdr = next( - (h for h in SINGLETON_HEADERS if len(headers.getall(h, ())) > 1), - None, - ) - if bad_hdr is not None: - raise BadHttpMessage(f"Duplicate '{bad_hdr}' header found.") - if self._cparser.type == cparser.HTTP_REQUEST: h_upg = headers.get("upgrade", "") allowed = upgrade and h_upg.isascii() and h_upg.lower() in ALLOWED_UPGRADES @@ -695,6 +698,7 @@ cdef class HttpResponseParser(HttpParser): cparser.llhttp_set_lenient_headers(self._cparser, 1) cparser.llhttp_set_lenient_optional_cr_before_lf(self._cparser, 1) cparser.llhttp_set_lenient_spaces_after_chunk_size(self._cparser, 1) + self._lax = True cdef object _on_status_complete(self): if self._buf: @@ -708,6 +712,7 @@ cdef int cb_on_message_begin(cparser.llhttp_t* parser) except -1: pyparser._started = True pyparser._headers = CIMultiDict() + pyparser._seen_singletons = set() pyparser._raw_headers = [] PyByteArray_Resize(pyparser._buf, 0) pyparser._path = None diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index a7fec7f01..4d7931ef6 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -87,6 +87,26 @@ VERSRE: Final[Pattern[str]] = re.compile(r"HTTP/(\d)\.(\d)", re.ASCII) DIGITS: Final[Pattern[str]] = re.compile(r"\d+", re.ASCII) HEXDIGITS: Final[Pattern[bytes]] = re.compile(rb"[0-9a-fA-F]+") +# RFC 9110 singleton headers — duplicates are rejected in strict mode. +# In lax mode (response parser default), the check is skipped entirely +# since real-world servers (e.g. Google APIs, Werkzeug) commonly send +# duplicate headers like Content-Type or Server. +# Lowercased for case-insensitive matching against wire names. +SINGLETON_HEADERS: Final[frozenset[str]] = frozenset( + { + "content-length", + "content-location", + "content-range", + "content-type", + "etag", + "host", + "max-forwards", + "server", + "transfer-encoding", + "user-agent", + } +) + class RawRequestMessage(NamedTuple): method: str @@ -216,6 +236,8 @@ class HeadersParser: elif _FIELD_VALUE_FORBIDDEN_CTL_RE.search(value): raise InvalidHeader(bvalue) + if not self._lax and name in headers and name.lower() in SINGLETON_HEADERS: + raise BadHttpMessage(f"Duplicate '{name}' header found.") headers.add(name, value) raw_headers.append((bname, bvalue)) @@ -526,24 +548,6 @@ class HttpParser(abc.ABC, Generic[_MsgT]): upgrade = False chunked = False - # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-6 - # https://www.rfc-editor.org/rfc/rfc9110.html#name-collected-abnf - singletons = ( - hdrs.CONTENT_LENGTH, - hdrs.CONTENT_LOCATION, - hdrs.CONTENT_RANGE, - hdrs.CONTENT_TYPE, - hdrs.ETAG, - hdrs.HOST, - hdrs.MAX_FORWARDS, - hdrs.SERVER, - hdrs.TRANSFER_ENCODING, - hdrs.USER_AGENT, - ) - bad_hdr = next((h for h in singletons if len(headers.getall(h, ())) > 1), None) - if bad_hdr is not None: - raise BadHttpMessage(f"Duplicate '{bad_hdr}' header found.") - # keep-alive conn = headers.get(hdrs.CONNECTION) if conn: diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index b0a282f3e..3e1f7dfaa 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -267,32 +267,76 @@ def test_content_length_transfer_encoding(parser: Any) -> None: "hdr", ( "Content-Length", + "Host", + "Transfer-Encoding", + ), +) +def test_duplicate_singleton_header_rejected( + parser: HttpRequestParser, hdr: str +) -> None: + val1, val2 = ("1", "2") if hdr == "Content-Length" else ("value1", "value2") + text = ( + f"GET /test HTTP/1.1\r\n" + f"Host: example.com\r\n" + f"{hdr}: {val1}\r\n" + f"{hdr}: {val2}\r\n" + "\r\n" + ).encode() + with pytest.raises(http_exceptions.BadHttpMessage, match="Duplicate"): + parser.feed_data(text) + + +@pytest.mark.parametrize( + "hdr", + ( "Content-Location", "Content-Range", "Content-Type", "ETag", - "Host", "Max-Forwards", "Server", - "Transfer-Encoding", "User-Agent", ), ) -def test_duplicate_singleton_header_rejected( +def test_duplicate_non_security_singleton_header_rejected_strict( parser: HttpRequestParser, hdr: str ) -> None: - val1, val2 = ("1", "2") if hdr == "Content-Length" else ("value1", "value2") + """Non-security singletons are rejected in strict mode (requests).""" text = ( f"GET /test HTTP/1.1\r\n" f"Host: example.com\r\n" - f"{hdr}: {val1}\r\n" - f"{hdr}: {val2}\r\n" - f"\r\n" + f"{hdr}: value1\r\n" + f"{hdr}: value2\r\n" + "\r\n" ).encode() with pytest.raises(http_exceptions.BadHttpMessage, match="Duplicate"): parser.feed_data(text) +@pytest.mark.parametrize( + "hdr", + ( + # Content-Length is excluded because llhttp rejects duplicates + # at the C level before our singleton check runs. + "Content-Location", + "Content-Range", + "Content-Type", + "ETag", + "Max-Forwards", + "Server", + "Transfer-Encoding", + "User-Agent", + ), +) +def test_duplicate_singleton_header_accepted_in_lax_mode( + response: HttpResponseParser, hdr: str +) -> None: + """All singleton duplicates are accepted in lax mode (response parser default).""" + text = (f"HTTP/1.1 200 OK\r\n{hdr}: value1\r\n{hdr}: value2\r\n\r\n").encode() + messages, upgrade, tail = response.feed_data(text) + assert len(messages) == 1 + + def test_duplicate_host_header_rejected(parser: HttpRequestParser) -> None: text = ( b"GET /admin HTTP/1.1\r\n" @@ -304,6 +348,45 @@ def test_duplicate_host_header_rejected(parser: HttpRequestParser) -> None: parser.feed_data(text) +@pytest.mark.parametrize( + ("hdr1", "hdr2"), + ( + ("content-length", "Content-Length"), + ("Content-Length", "content-length"), + ("transfer-encoding", "Transfer-Encoding"), + ("Transfer-Encoding", "transfer-encoding"), + ), +) +def test_duplicate_singleton_header_different_casing_rejected( + parser: HttpRequestParser, hdr1: str, hdr2: str +) -> None: + """Singleton check must be case-insensitive per RFC 9110.""" + val1, val2 = ("1", "2") if "content-length" in hdr1.lower() else ("v1", "v2") + text = ( + f"GET /test HTTP/1.1\r\n" + f"Host: example.com\r\n" + f"{hdr1}: {val1}\r\n" + f"{hdr2}: {val2}\r\n" + "\r\n" + ).encode() + with pytest.raises(http_exceptions.BadHttpMessage, match="Duplicate"): + parser.feed_data(text) + + +def test_duplicate_host_header_different_casing_rejected( + parser: HttpRequestParser, +) -> None: + """Duplicate Host with different casing must also be rejected.""" + text = ( + b"GET /test HTTP/1.1\r\n" + b"host: evil.example\r\n" + b"Host: good.example\r\n" + b"\r\n" + ) + with pytest.raises(http_exceptions.BadHttpMessage, match="Duplicate"): + parser.feed_data(text) + + def test_bad_chunked_py(loop: Any, protocol: Any) -> None: """Test that invalid chunked encoding doesn't allow content-length to be used.""" parser = HttpRequestParserPy( -- 2.35.6