mirror of
https://github.com/openembedded/meta-openembedded.git
synced 2026-09-09 06:40:17 +00:00
This patch applies the upstream fix for CVE-2026-59939 as referenced in [2], using the upstream commit identified in [1]. [1] https://github.com/httplib2/httplib2/commit/87581ad6cf752fe3da2090c59058261d2d00a427 [2] https://github.com/httplib2/httplib2/security/advisories/GHSA-j5g9-f88f-gfj3 Signed-off-by: Darsh Kelaiya <dkelaiya@cisco.com> Signed-off-by: Anuj Mittal <anuj.mittal@oss.qualcomm.com>
746 lines
26 KiB
Diff
746 lines
26 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: Sergey Shepelev <temotor@gmail.com>
|
|
Date: Wed, 24 Jun 2026 12:34:50 +0300
|
|
Subject: [PATCH] decompression limited by size and ratio; require python 3.8+
|
|
|
|
CVE: CVE-2026-59939
|
|
Upstream-Status: Backport [https://github.com/httplib2/httplib2/commit/87581ad6cf752fe3da2090c59058261d2d00a427]
|
|
|
|
Backport Changes:
|
|
- Place httplib2 sources under python3/httplib2 to match the
|
|
Scarthgap 0.22.0 source layout.
|
|
- Cap each zlib decompression call with max_length derived from the
|
|
remaining hard and ratio allowance, requeue unconsumed_tail, and
|
|
enforce both limits immediately after each returned output chunk.
|
|
- Reject empty gzip and deflate bodies through the existing
|
|
FailedToDecompressContent contract.
|
|
- Add constrained-memory and empty encoded-body regression tests.
|
|
- Keep Scarthgap's Python version metadata; the upstream Python 3.8
|
|
minimum is unrelated to the security fix.
|
|
- Omit .github/workflows updates because those CI files are absent
|
|
from the extracted source and are not part of the security fix.
|
|
|
|
(cherry picked from commit 87581ad6cf752fe3da2090c59058261d2d00a427)
|
|
Signed-off-by: Darsh Kelaiya <dkelaiya@cisco.com>
|
|
---
|
|
README.md | 37 ++++++--
|
|
python3/httplib2/__init__.py | 71 +++++++++++---
|
|
python3/httplib2/decode.py | 209 ++++++++++++++++++++++++++++++++++++++++
|
|
setup.cfg | 2 +-
|
|
tests/__init__.py | 8 ++
|
|
tests/test_encoding.py | 221 +++++++++++++++++++++++++++++++++++++++++--
|
|
6 files changed, 517 insertions(+), 31 deletions(-)
|
|
|
|
diff --git a/README.md b/README.md
|
|
index 6193699..99a0d3b 100644
|
|
--- a/README.md
|
|
+++ b/README.md
|
|
@@ -10,13 +10,13 @@ If you want to help this project by bug report or code change, [contribution gui
|
|
|
|
HTTPS support is only available if the socket module was
|
|
compiled with SSL support.
|
|
-
|
|
+
|
|
### Keep-Alive
|
|
|
|
Supports HTTP 1.1 Keep-Alive, keeping the socket open and
|
|
performing multiple requests over the same connection if
|
|
possible.
|
|
-
|
|
+
|
|
### Authentication
|
|
|
|
The following three types of HTTP Authentication are
|
|
@@ -31,26 +31,26 @@ supported. These can be used over both HTTP and HTTPS.
|
|
The module can optionally operate with a private cache that
|
|
understands the Cache-Control: header and uses both the ETag
|
|
and Last-Modified cache validators.
|
|
-
|
|
+
|
|
### All Methods
|
|
|
|
The module can handle any HTTP request method, not just GET
|
|
and POST.
|
|
-
|
|
+
|
|
### Redirects
|
|
|
|
Automatically follows 3XX redirects on GETs.
|
|
-
|
|
+
|
|
### Compression
|
|
|
|
Handles both 'deflate' and 'gzip' types of compression.
|
|
-
|
|
+
|
|
### Lost update support
|
|
|
|
Automatically adds back ETags into PUT requests to resources
|
|
we have already cached. This implements Section 3.2 of
|
|
Detecting the Lost Update Problem Using Unreserved Checkout.
|
|
-
|
|
+
|
|
### Unit Tested
|
|
|
|
A large and growing set of unit tests.
|
|
@@ -113,3 +113,26 @@ More example usage can be found at:
|
|
|
|
* https://github.com/httplib2/httplib2/wiki/Examples
|
|
* https://github.com/httplib2/httplib2/wiki/Examples-Python3
|
|
+
|
|
+
|
|
+### Decompression Limits
|
|
+
|
|
+To mitigate denial-of-service risks from maliciously crafted compressed responses, the library enforces configurable limits during decompression. Limits are checked in fixed order: **hard limit** → **safe limit** → **ratio**.
|
|
+
|
|
+- **hard limit**
|
|
+ Absolute maximum decompressed output size (bytes). Exceeding it raises `DecodeLimitError`. Default: `10 GiB`.
|
|
+- **safe limit**
|
|
+ Output size below which the ratio check is skipped (avoids false positives on small payloads). Default: `10 MiB`.
|
|
+- **ratio**
|
|
+ Maximum allowed inflation factor (`output_bytes ÷ consumed_input_bytes`). Once output exceeds `safe_limit`, the ratio is enforced. Default: `100`.
|
|
+- **chunk size**
|
|
+ Internal processing chunk size in bytes (affects granularity of limit checks). Default: `65536` (64 KiB).
|
|
+
|
|
+Configuration priority (highest first):
|
|
+1. `Http()` constructor arguments: `decode_limit_hard`, `decode_limit_safe`, `decode_limit_ratio`, `decode_limit_chunk`
|
|
+2. Environment variables: `httplib2_decode_limit_hard`, `httplib2_decode_limit_safe`, `httplib2_decode_limit_ratio`, `httplib2_decode_limit_chunk` (case-insensitive, uppercase also accepted)
|
|
+3. Library defaults (listed above)
|
|
+
|
|
+Example:
|
|
+```python
|
|
+h = Http(decode_limit_hard=50_000_000, decode_limit_ratio=50)
|
|
diff --git a/python3/httplib2/__init__.py b/python3/httplib2/__init__.py
|
|
index 723a63c..ca2868a 100644
|
|
--- a/python3/httplib2/__init__.py
|
|
+++ b/python3/httplib2/__init__.py
|
|
@@ -1,6 +1,10 @@
|
|
# -*- coding: utf-8 -*-
|
|
"""Small, fast HTTP client library for Python."""
|
|
|
|
+import functools
|
|
+
|
|
+from httplib2.decode import ZlibDecoder, DecoderProtocol, LimitDecoder, DeflateDecoder
|
|
+
|
|
__author__ = "Joe Gregorio (joe@bitworking.org)"
|
|
__copyright__ = "Copyright 2006, Joe Gregorio"
|
|
__contributors__ = [
|
|
@@ -386,26 +390,27 @@ def _entry_disposition(response_headers, request_headers):
|
|
return retval
|
|
|
|
|
|
-def _decompressContent(response, new_content):
|
|
+def _decompressContent(response, new_content, limit_kwargs):
|
|
content = new_content
|
|
+ encoding_header = "content-encoding"
|
|
+ encoding = response.get(encoding_header, None)
|
|
+ limit_wrap = functools.partial(LimitDecoder, **limit_kwargs)
|
|
try:
|
|
- encoding = response.get("content-encoding", None)
|
|
- if encoding in ["gzip", "deflate"]:
|
|
- if encoding == "gzip":
|
|
- content = gzip.GzipFile(fileobj=io.BytesIO(new_content)).read()
|
|
- if encoding == "deflate":
|
|
- try:
|
|
- content = zlib.decompress(content, zlib.MAX_WBITS)
|
|
- except (IOError, zlib.error):
|
|
- content = zlib.decompress(content, -zlib.MAX_WBITS)
|
|
+ if encoding in ["gzip", "deflate", "zlib"]:
|
|
+ if not new_content:
|
|
+ raise zlib.error("empty compressed response")
|
|
+ try:
|
|
+ content = limit_wrap(ZlibDecoder()).consume_bytes(new_content, 0)
|
|
+ except (IOError, zlib.error):
|
|
+ content = limit_wrap(DeflateDecoder()).consume_bytes(new_content, 0)
|
|
response["content-length"] = str(len(content))
|
|
# Record the historical presence of the encoding in a way the won't interfere.
|
|
- response["-content-encoding"] = response["content-encoding"]
|
|
- del response["content-encoding"]
|
|
+ response["-content-encoding"] = response.pop(encoding_header)
|
|
except (IOError, zlib.error):
|
|
content = ""
|
|
raise FailedToDecompressContent(
|
|
- _("Content purported to be compressed with %s but failed to decompress.") % response.get("content-encoding"),
|
|
+ _("Content purported to be compressed with %s but failed to decompress.")
|
|
+ % encoding,
|
|
response,
|
|
content,
|
|
)
|
|
@@ -1232,6 +1237,10 @@ class Http(object):
|
|
disable_ssl_certificate_validation=False,
|
|
tls_maximum_version=None,
|
|
tls_minimum_version=None,
|
|
+ decode_limit_hard=None,
|
|
+ decode_limit_safe=None,
|
|
+ decode_limit_ratio=None,
|
|
+ decode_limit_chunk=None,
|
|
):
|
|
"""If 'cache' is a string then it is used as a directory name for
|
|
a disk cache. Otherwise it must be an object that supports the
|
|
@@ -1258,6 +1267,11 @@ class Http(object):
|
|
|
|
tls_maximum_version / tls_minimum_version require Python 3.7+ /
|
|
OpenSSL 1.1.0g+. A value of "TLSv1_3" requires OpenSSL 1.1.1+.
|
|
+
|
|
+ `decode_limit_{hard,safe,ratio,chunk}` options configure `httplib2.decode.LimitDecoder` in attempt order:
|
|
+ - Http() argument - top priority
|
|
+ - environment httplib2_decode_limit_{hard,safe,ratio,chunk}
|
|
+ - LimitDecoder defaults - least priority
|
|
"""
|
|
self.proxy_info = proxy_info
|
|
self.ca_certs = ca_certs
|
|
@@ -1306,6 +1320,22 @@ class Http(object):
|
|
# Keep Authorization: headers on a redirect.
|
|
self.forward_authorization_headers = False
|
|
|
|
+ limit_kwargs = dict(
|
|
+ hard_limit=try_value_or_env(
|
|
+ int, decode_limit_hard, "httplib2_decode_limit_hard"
|
|
+ ),
|
|
+ safe_limit=try_value_or_env(
|
|
+ int, decode_limit_safe, "httplib2_decode_limit_safe"
|
|
+ ),
|
|
+ ratio=try_value_or_env(
|
|
+ float, decode_limit_ratio, "httplib2_decode_limit_ratio"
|
|
+ ),
|
|
+ chunk_size=try_value_or_env(
|
|
+ int, decode_limit_chunk, "httplib2_decode_limit_chunk"
|
|
+ ),
|
|
+ )
|
|
+ self.limit_kwargs = {k: v for k, v in limit_kwargs.items() if v is not None}
|
|
+
|
|
def close(self):
|
|
"""Close persistent connections, clear sensitive data.
|
|
Not thread-safe, requires external synchronization against concurrent requests.
|
|
@@ -1425,7 +1455,7 @@ class Http(object):
|
|
content = response.read()
|
|
response = Response(response)
|
|
if method != "HEAD":
|
|
- content = _decompressContent(response, content)
|
|
+ content = _decompressContent(response, content, self.limit_kwargs)
|
|
|
|
break
|
|
return (response, content)
|
|
@@ -1797,3 +1827,16 @@ class Response(dict):
|
|
return self
|
|
else:
|
|
raise AttributeError(name)
|
|
+
|
|
+
|
|
+def try_value_or_env(to, value, env_key, default=None):
|
|
+ candidates = (value, os.environ.get(env_key), os.environ.get(env_key.upper()))
|
|
+ # same as `to(x1) or to(x2) or to(x3)` except accepting falsey values like 0
|
|
+ for x in candidates:
|
|
+ if x is None:
|
|
+ continue
|
|
+ try:
|
|
+ return to(x)
|
|
+ except ValueError:
|
|
+ pass
|
|
+ return default
|
|
diff --git a/python3/httplib2/decode.py b/python3/httplib2/decode.py
|
|
new file mode 100644
|
|
index 0000000..588cf42
|
|
--- /dev/null
|
|
+++ b/python3/httplib2/decode.py
|
|
@@ -0,0 +1,209 @@
|
|
+from typing import Protocol
|
|
+import zlib
|
|
+
|
|
+
|
|
+class DecodeRatioError(Exception):
|
|
+ """Output-to-input amplification ratio exceeded the configured limit."""
|
|
+
|
|
+
|
|
+class DecodeLimitError(Exception):
|
|
+ """Total output length exceeded the hard limit."""
|
|
+
|
|
+
|
|
+class DecoderProtocol(Protocol):
|
|
+ @property
|
|
+ def needs_input(self) -> bool:
|
|
+ ...
|
|
+
|
|
+ def decode(self, b: bytes, max_length: int = 0) -> bytes:
|
|
+ ...
|
|
+
|
|
+ @property
|
|
+ def unconsumed_tail(self) -> bytes:
|
|
+ ...
|
|
+
|
|
+ def flush(self) -> bytes:
|
|
+ ...
|
|
+
|
|
+ def consume_bytes(self, data: bytes, chunk_size: int = 64 << 10) -> bytes:
|
|
+ out = bytearray()
|
|
+ if not data:
|
|
+ return self.flush()
|
|
+ if chunk_size == 0:
|
|
+ chunk_size = len(data)
|
|
+ for i in range(0, len(data), chunk_size):
|
|
+ chunk = data[i : i + chunk_size]
|
|
+ out.extend(self.decode(chunk))
|
|
+ out.extend(self.flush())
|
|
+ return bytes(out)
|
|
+
|
|
+
|
|
+class ZlibDecoder(DecoderProtocol):
|
|
+ """
|
|
+ Thin wrapper around zlib.Decompressor conforming to the Decoder interface.
|
|
+
|
|
+ When max_length caps output, zlib exposes input that still needs processing
|
|
+ through unconsumed_tail.
|
|
+ """
|
|
+
|
|
+ __slots__ = ("_decoder",)
|
|
+
|
|
+ WBITS_DEFLATE = -15
|
|
+ WBITS_ZLIB = 15
|
|
+ WBITS_GZIP = 15 | 16
|
|
+ WBITS_AUTO_GZIP_ZLIB = 15 | 32 # but not deflate
|
|
+
|
|
+ def __init__(self, wbits: int = WBITS_AUTO_GZIP_ZLIB):
|
|
+ self._decoder: zlib._Decompress | None = zlib.decompressobj(wbits)
|
|
+
|
|
+ @property
|
|
+ def needs_input(self) -> bool:
|
|
+ if self._decoder is None:
|
|
+ raise RuntimeError("used after flush()")
|
|
+ return not self._decoder.unconsumed_tail and not self._decoder.eof
|
|
+
|
|
+ def decode(self, b: bytes, max_length: int = 0) -> bytes:
|
|
+ if self._decoder is None:
|
|
+ raise RuntimeError("used after flush()")
|
|
+ return self._decoder.decompress(b, max_length)
|
|
+
|
|
+ @property
|
|
+ def unconsumed_tail(self) -> bytes:
|
|
+ if self._decoder is None:
|
|
+ raise RuntimeError("used after flush()")
|
|
+ return self._decoder.unconsumed_tail
|
|
+
|
|
+ def flush(self) -> bytes:
|
|
+ if self._decoder is None:
|
|
+ raise RuntimeError("used after flush()")
|
|
+ result = self._decoder.flush()
|
|
+ self._decoder = None
|
|
+ return result
|
|
+
|
|
+
|
|
+def DeflateDecoder() -> ZlibDecoder:
|
|
+ return ZlibDecoder(ZlibDecoder.WBITS_DEFLATE)
|
|
+
|
|
+
|
|
+class LimitDecoder(DecoderProtocol):
|
|
+ __slots__ = (
|
|
+ "_decoder",
|
|
+ "_ratio",
|
|
+ "_chunk_size",
|
|
+ "_safe_limit",
|
|
+ "_hard_limit",
|
|
+ "_consumed_length",
|
|
+ "_output_length",
|
|
+ "_input_buffer",
|
|
+ "_flushed",
|
|
+ )
|
|
+
|
|
+ def __init__(
|
|
+ self,
|
|
+ decoder: DecoderProtocol,
|
|
+ ratio: float = 100,
|
|
+ chunk_size: int = 64 << 10,
|
|
+ safe_limit: int = 10 << 20,
|
|
+ hard_limit: int = 10 << 30,
|
|
+ ) -> None:
|
|
+ if ratio < 0:
|
|
+ raise ValueError(f"LimitDecoder() ratio={ratio} expected >= 0")
|
|
+ if chunk_size < 0:
|
|
+ raise ValueError(f"LimitDecoder() chunk_size={chunk_size} expected >= 0")
|
|
+ if safe_limit < 0:
|
|
+ raise ValueError(f"LimitDecoder() safe_limit={safe_limit} expected >= 0")
|
|
+ if hard_limit < 0:
|
|
+ raise ValueError(f"LimitDecoder() safe_limit={safe_limit} expected >= 0")
|
|
+
|
|
+ self._decoder: DecoderProtocol = decoder
|
|
+ self._ratio: float = ratio
|
|
+ self._chunk_size: int = chunk_size
|
|
+ self._safe_limit: int = safe_limit
|
|
+ self._hard_limit: int = hard_limit
|
|
+ self._consumed_length: int = 0
|
|
+ self._output_length: int = 0
|
|
+ self._input_buffer: bytearray = bytearray()
|
|
+ self._flushed: bool = False
|
|
+
|
|
+ def _check_limits(self) -> None:
|
|
+ if (self._hard_limit > 0) and (self._output_length > self._hard_limit):
|
|
+ raise DecodeLimitError(f"Output length {self._output_length} exceeds hard limit {self._hard_limit}")
|
|
+ if (self._safe_limit > 0) and (self._output_length < self._safe_limit):
|
|
+ return
|
|
+ if (self._ratio > 0) and (self._output_length > self._consumed_length * self._ratio):
|
|
+ actual_ratio = self._output_length / self._consumed_length if self._consumed_length > 0 else float("inf")
|
|
+ raise DecodeRatioError(
|
|
+ f"Amplification ratio {actual_ratio:.1f} ({self._output_length}/{self._consumed_length})"
|
|
+ f" exceeds limit {self._ratio}"
|
|
+ )
|
|
+
|
|
+ def _decode_max_length(self, input_length: int) -> int:
|
|
+ limits = []
|
|
+ if self._hard_limit > 0:
|
|
+ limits.append(self._hard_limit)
|
|
+ if self._ratio > 0:
|
|
+ ratio_limit = int(
|
|
+ (self._consumed_length + input_length) * self._ratio
|
|
+ )
|
|
+ if self._safe_limit > 0:
|
|
+ ratio_limit = max(ratio_limit, self._safe_limit - 1)
|
|
+ limits.append(ratio_limit)
|
|
+ if not limits:
|
|
+ return 0
|
|
+ return max(1, min(limits) - self._output_length + 1)
|
|
+
|
|
+ @property
|
|
+ def needs_input(self) -> bool:
|
|
+ return self._decoder.needs_input
|
|
+
|
|
+ def decode(self, b: bytes, max_length: int = 0) -> bytes:
|
|
+ if self._flushed:
|
|
+ raise RuntimeError("decode() called after flush()")
|
|
+ output = self._pump(b)
|
|
+ return bytes(output)
|
|
+
|
|
+ def flush(self) -> bytes:
|
|
+ if self._flushed:
|
|
+ raise RuntimeError("flush() called more than once")
|
|
+ self._flushed = True
|
|
+
|
|
+ output = self._pump(b"")
|
|
+
|
|
+ data = self._decoder.flush()
|
|
+ output.extend(data)
|
|
+ self._output_length += len(data)
|
|
+ self._check_limits()
|
|
+
|
|
+ return bytes(output)
|
|
+
|
|
+ def _pump(self, b: bytes) -> bytearray:
|
|
+ self._input_buffer.extend(b)
|
|
+
|
|
+ output = bytearray()
|
|
+ drain = False
|
|
+ while True:
|
|
+ if self._input_buffer:
|
|
+ chunk = bytes(self._input_buffer[: self._chunk_size])
|
|
+ del self._input_buffer[: self._chunk_size]
|
|
+ elif drain:
|
|
+ chunk = b""
|
|
+ else:
|
|
+ break
|
|
+
|
|
+ max_length = self._decode_max_length(len(chunk))
|
|
+ data = self._decoder.decode(chunk, max_length)
|
|
+ tail = self._decoder.unconsumed_tail
|
|
+ consumed = len(chunk) - len(tail)
|
|
+ self._consumed_length += consumed
|
|
+ if tail:
|
|
+ self._input_buffer[:0] = tail
|
|
+
|
|
+ output.extend(data)
|
|
+ self._output_length += len(data)
|
|
+ self._check_limits()
|
|
+
|
|
+ drain = bool(max_length and len(data) == max_length)
|
|
+ if not data and not consumed and not tail:
|
|
+ break
|
|
+
|
|
+ return output
|
|
diff --git a/setup.cfg b/setup.cfg
|
|
index fa392d1..93641b0 100644
|
|
--- a/setup.cfg
|
|
+++ b/setup.cfg
|
|
@@ -2,7 +2,7 @@
|
|
|
|
[flake8]
|
|
exclude = *.egg*,.env,.git,.tox,_*,build*,dist*,venv*,python2/,python3/
|
|
-ignore = E261,E731,W503
|
|
+ignore = E203,E261,E731,W503
|
|
max-line-length = 121
|
|
|
|
[tool:pytest]
|
|
diff --git a/tests/__init__.py b/tests/__init__.py
|
|
index b5c76a2..d0f4538 100644
|
|
--- a/tests/__init__.py
|
|
+++ b/tests/__init__.py
|
|
@@ -818,3 +818,11 @@ def rebuild_uri(old, scheme=_missing, netloc=_missing, host=_missing, port=_miss
|
|
path = u.path
|
|
new = (scheme, netloc, path) + u[3:]
|
|
return urllib.parse.urlunsplit(new)
|
|
+
|
|
+
|
|
+# remove when python version is raised to 3.9+
|
|
+try:
|
|
+ randbytes = random.randbytes
|
|
+except AttributeError:
|
|
+ def randbytes(n):
|
|
+ return random.getrandbits(n * 8).to_bytes(n, "little")
|
|
diff --git a/tests/test_encoding.py b/tests/test_encoding.py
|
|
index 1d9770a..7431e24 100644
|
|
--- a/tests/test_encoding.py
|
|
+++ b/tests/test_encoding.py
|
|
@@ -1,13 +1,17 @@
|
|
+import subprocess
|
|
+import sys
|
|
+
|
|
+import pytest
|
|
+
|
|
import httplib2
|
|
+from httplib2.decode import LimitDecoder, ZlibDecoder, DecodeLimitError, DecodeRatioError
|
|
import tests
|
|
|
|
|
|
def test_gzip_head():
|
|
# Test that we don't try to decompress a HEAD response
|
|
http = httplib2.Http()
|
|
- response = tests.http_response_bytes(
|
|
- headers={"content-encoding": "gzip", "content-length": 42}
|
|
- )
|
|
+ response = tests.http_response_bytes(headers={"content-encoding": "gzip", "content-length": 42})
|
|
with tests.server_const_bytes(response) as uri:
|
|
response, content = http.request(uri, "HEAD")
|
|
assert response.status == 200
|
|
@@ -48,9 +52,7 @@ def test_gzip_malformed_response():
|
|
http = httplib2.Http()
|
|
# Test that we raise a good exception when the gzip fails
|
|
http.force_exception_to_status_code = False
|
|
- response = tests.http_response_bytes(
|
|
- headers={"content-encoding": "gzip"}, body=b"obviously not compressed"
|
|
- )
|
|
+ response = tests.http_response_bytes(headers={"content-encoding": "gzip"}, body=b"obviously not compressed")
|
|
with tests.server_const_bytes(response, request_count=2) as uri:
|
|
with tests.assert_raises(httplib2.FailedToDecompressContent):
|
|
http.request(uri, "GET")
|
|
@@ -82,9 +84,7 @@ def test_deflate_malformed_response():
|
|
# Test that we raise a good exception when the deflate fails
|
|
http = httplib2.Http()
|
|
http.force_exception_to_status_code = False
|
|
- response = tests.http_response_bytes(
|
|
- headers={"content-encoding": "deflate"}, body=b"obviously not compressed"
|
|
- )
|
|
+ response = tests.http_response_bytes(headers={"content-encoding": "deflate"}, body=b"obviously not compressed")
|
|
with tests.server_const_bytes(response, request_count=2) as uri:
|
|
with tests.assert_raises(httplib2.FailedToDecompressContent):
|
|
http.request(uri, "GET")
|
|
@@ -97,6 +97,18 @@ def test_deflate_malformed_response():
|
|
assert response.reason.startswith("Content purported")
|
|
|
|
|
|
+@pytest.mark.parametrize("encoding", ("gzip", "deflate"))
|
|
+def test_empty_encoded_response(encoding):
|
|
+ http = httplib2.Http()
|
|
+ http.force_exception_to_status_code = False
|
|
+ response = tests.http_response_bytes(
|
|
+ headers={"content-encoding": encoding}, body=b""
|
|
+ )
|
|
+ with tests.server_const_bytes(response) as uri:
|
|
+ with tests.assert_raises(httplib2.FailedToDecompressContent):
|
|
+ http.request(uri, "GET")
|
|
+
|
|
+
|
|
def test_zlib_get():
|
|
# Test that we support zlib compression
|
|
http = httplib2.Http()
|
|
@@ -110,3 +122,194 @@ def test_zlib_get():
|
|
assert "content-encoding" not in response
|
|
assert int(response["content-length"]) == len(b"properly compressed")
|
|
assert content == b"properly compressed"
|
|
+
|
|
+
|
|
+def test_gzip_excess_ratio():
|
|
+ http = httplib2.Http()
|
|
+ original = b"\x00" * (50 << 20) # 50 MiB to ~50 KiB
|
|
+ response = tests.http_response_bytes(
|
|
+ headers={"content-encoding": "gzip"},
|
|
+ body=tests.gzip_compress(original),
|
|
+ )
|
|
+ with tests.server_const_bytes(response) as uri:
|
|
+ try:
|
|
+ http.request(uri, "GET")
|
|
+ assert False, "expected DecodeRatioError"
|
|
+ except DecodeRatioError:
|
|
+ pass
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize("safe_limit", (0, 1000, 20000))
|
|
+def test_limitdecoder_normal_decompression_no_limits(safe_limit):
|
|
+ """Standard decompression of random data should pass with any safe_limit"""
|
|
+ original = tests.randbytes(10 << 10)
|
|
+ compressed = tests.zlib_compress(original)
|
|
+
|
|
+ decoder = LimitDecoder(
|
|
+ ZlibDecoder(),
|
|
+ ratio=10,
|
|
+ safe_limit=safe_limit,
|
|
+ hard_limit=len(original) + 1,
|
|
+ )
|
|
+ result = decoder.consume_bytes(compressed)
|
|
+ assert result == original
|
|
+ assert decoder._output_length == len(original)
|
|
+
|
|
+
|
|
+def test_limitdecoder_normal_rechunking():
|
|
+ """Passing a massive single chunk should be re-chunked internally without error"""
|
|
+ original = b"\x00" * (10 << 20)
|
|
+ compressed = tests.zlib_compress(original)
|
|
+ assert len(compressed) > 2000
|
|
+
|
|
+ decoder = LimitDecoder(
|
|
+ ZlibDecoder(),
|
|
+ ratio=2000,
|
|
+ chunk_size=512,
|
|
+ safe_limit=0,
|
|
+ hard_limit=len(original) + 1,
|
|
+ )
|
|
+ result = decoder.consume_bytes(compressed, chunk_size=0)
|
|
+ assert result == original
|
|
+ assert decoder._consumed_length == len(compressed)
|
|
+
|
|
+
|
|
+def test_limitdecoder_amplification_ratio_exceeded():
|
|
+ """High ratio should trigger DecodeRatioError above safe_limit"""
|
|
+ original = b"\x00" * (1 << 20)
|
|
+ compressed = tests.zlib_compress(original)
|
|
+
|
|
+ decoder = LimitDecoder(
|
|
+ ZlibDecoder(),
|
|
+ ratio=10,
|
|
+ chunk_size=512,
|
|
+ safe_limit=0,
|
|
+ hard_limit=len(original) + 1,
|
|
+ )
|
|
+ try:
|
|
+ decoder.consume_bytes(compressed, chunk_size=0)
|
|
+ assert False, "expected DecodeRatioError"
|
|
+ except DecodeRatioError:
|
|
+ pass
|
|
+ assert 0 < decoder._consumed_length <= 512
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize("ratio", (0, 10, 1000))
|
|
+def test_limitdecoder_hard_limit_exceeded(ratio):
|
|
+ """Output exceeding hard_limit must trigger DecodeLimitError regardless of ratio"""
|
|
+ original = b"\x00" * (10 << 10)
|
|
+ compressed = tests.zlib_compress(original)
|
|
+
|
|
+ decoder = LimitDecoder(
|
|
+ ZlibDecoder(),
|
|
+ ratio=ratio,
|
|
+ safe_limit=len(original) + 1,
|
|
+ hard_limit=len(original) - 1,
|
|
+ )
|
|
+ try:
|
|
+ decoder.consume_bytes(compressed)
|
|
+ assert False, "expected DecodeLimitError"
|
|
+ except DecodeLimitError:
|
|
+ pass
|
|
+
|
|
+
|
|
+def test_limitdecoder_bounds_allocation_before_hard_limit():
|
|
+ code = r"""
|
|
+import os
|
|
+import resource
|
|
+import zlib
|
|
+
|
|
+from httplib2.decode import DecodeLimitError, DecodeRatioError, LimitDecoder
|
|
+from httplib2.decode import ZlibDecoder
|
|
+
|
|
+compressor = zlib.compressobj(wbits=31)
|
|
+compressed = b"".join(
|
|
+ compressor.compress(b"\x00" * (1 << 20)) for _ in range(50)
|
|
+)
|
|
+compressed += compressor.flush()
|
|
+page_size = os.sysconf("SC_PAGE_SIZE")
|
|
+with open("/proc/self/statm", "r") as statm:
|
|
+ current_vms = int(statm.read().split()[0]) * page_size
|
|
+_, hard = resource.getrlimit(resource.RLIMIT_AS)
|
|
+resource.setrlimit(resource.RLIMIT_AS, (current_vms + (30 << 20), hard))
|
|
+decoder = LimitDecoder(
|
|
+ ZlibDecoder(), ratio=100, safe_limit=0, hard_limit=1 << 20
|
|
+)
|
|
+try:
|
|
+ decoder.consume_bytes(compressed, chunk_size=0)
|
|
+except (DecodeLimitError, DecodeRatioError):
|
|
+ pass
|
|
+except MemoryError:
|
|
+ raise AssertionError("decompressor allocated past the configured limit")
|
|
+else:
|
|
+ raise AssertionError("expected a decompression limit error")
|
|
+"""
|
|
+ result = subprocess.run(
|
|
+ [sys.executable, "-c", code], capture_output=True, text=True
|
|
+ )
|
|
+ assert result.returncode == 0, result.stderr
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize("ratio", (0, 10, 1000))
|
|
+def test_limitdecoder_safe_limit_bypass(ratio):
|
|
+ """Any ratio allowed if total output < safe_limit"""
|
|
+ original = b"\x00" * (10 << 10)
|
|
+ compressed = tests.zlib_compress(original)
|
|
+
|
|
+ decoder = LimitDecoder(
|
|
+ ZlibDecoder(),
|
|
+ ratio=ratio,
|
|
+ safe_limit=len(original) + 1,
|
|
+ hard_limit=len(original) + 1,
|
|
+ )
|
|
+ result = decoder.consume_bytes(compressed)
|
|
+ assert result == original
|
|
+ assert decoder._output_length == len(original)
|
|
+
|
|
+
|
|
+def test_limitdecoder_single_byte_feeding():
|
|
+ """Feeding compressed data 1 byte at a time should still decode correctly"""
|
|
+ original = tests.randbytes(10 << 10)
|
|
+ compressed = tests.zlib_compress(original)
|
|
+
|
|
+ decoder = LimitDecoder(
|
|
+ ZlibDecoder(),
|
|
+ ratio=10,
|
|
+ safe_limit=5 << 10,
|
|
+ hard_limit=len(original) + 1,
|
|
+ )
|
|
+ result = decoder.consume_bytes(compressed, chunk_size=1)
|
|
+ assert result == original
|
|
+
|
|
+
|
|
+def test_limitdecoder_invalid_argument():
|
|
+ checks = (
|
|
+ ("ratio", dict(ratio=-1)),
|
|
+ ("chunk_size", dict(chunk_size=-1)),
|
|
+ ("safe_limit", dict(safe_limit=-1)),
|
|
+ ("hard_limit", dict(hard_limit=-1)),
|
|
+ )
|
|
+ for name, check in checks:
|
|
+ zd = ZlibDecoder()
|
|
+ try:
|
|
+ LimitDecoder(zd, **check)
|
|
+ assert False, f"check={name} expected ValueError"
|
|
+ except ValueError as e:
|
|
+ assert "expected >= 0" in str(e).lower(), str(e)
|
|
+
|
|
+
|
|
+def test_zlibdecoder_invalid_after_flush():
|
|
+ checks = (
|
|
+ ("needs_input", lambda d: d.needs_input),
|
|
+ ("decode", lambda d: d.decode(b"")),
|
|
+ ("flush", lambda d: d.flush()),
|
|
+ )
|
|
+ for name, check in checks:
|
|
+ d = ZlibDecoder()
|
|
+ d.decode(tests.zlib_compress(b""))
|
|
+ d.flush()
|
|
+ try:
|
|
+ check(d)
|
|
+ assert False, f"check={name} expected RuntimeError"
|
|
+ except RuntimeError as e:
|
|
+ assert "used after flush" in str(e).lower(), str(e)
|