mirror of
https://github.com/openembedded/meta-openembedded.git
synced 2026-09-01 16:10:25 +00:00
Hetvi Thakar
34800488de
This patch applies the upstream fix for CVE-2026-54911 to ujson 5.12.1. The upstream fix commit is referenced in [1], and the public security advisory is referenced in [2]. [1] https://github.com/ultrajson/ultrajson/commit/169eaf36b1116fece5034ee79a7a0ef3f6deedcf [2] https://github.com/ultrajson/ultrajson/security/advisories/GHSA-3j69-69wj-xqx2 Signed-off-by: Hetvi Thakar <hthakar@cisco.com> Signed-off-by: Anuj Mittal <anuj.mittal@oss.qualcomm.com>
254 lines
11 KiB
Diff
254 lines
11 KiB
Diff
From 169eaf36b1116fece5034ee79a7a0ef3f6deedcf Mon Sep 17 00:00:00 2001
|
|
From: =?UTF-8?q?Br=C3=A9nainn=20Woodsend?= <bwoodsend@gmail.com>
|
|
Date: Fri, 24 Apr 2026 21:59:45 +0100
|
|
Subject: [PATCH] More UTF-8 validation for ujson.dumps(b"...",
|
|
reject_bytes=False)
|
|
|
|
* Fix off by one errors in detecting end of string mid sequence
|
|
* Add missing check for codepoints > max unicode
|
|
* Add missing check for bad continuation bytes
|
|
|
|
CVE: CVE-2026-54911
|
|
Upstream-Status: Backport [https://github.com/ultrajson/ultrajson/commit/169eaf36b1116fece5034ee79a7a0ef3f6deedcf]
|
|
|
|
(cherry picked from commit 169eaf36b1116fece5034ee79a7a0ef3f6deedcf)
|
|
Signed-off-by: Hetvi Thakar <hthakar@cisco.com>
|
|
---
|
|
src/ujson/lib/ultrajsondec.c | 6 +--
|
|
src/ujson/lib/ultrajsonenc.c | 46 ++++++++++++++++++----
|
|
tests/test_ujson.py | 75 ++++++++++++++++++++++++++++++++++++
|
|
3 files changed, 117 insertions(+), 10 deletions(-)
|
|
|
|
diff --git a/src/ujson/lib/ultrajsondec.c b/src/ujson/lib/ultrajsondec.c
|
|
index bccb0aaf..55833763 100644
|
|
--- a/src/ujson/lib/ultrajsondec.c
|
|
+++ b/src/ujson/lib/ultrajsondec.c
|
|
@@ -531,7 +531,7 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds
|
|
return SetError(ds, -1, "Invalid octet in UTF-8 sequence when decoding 'string'");
|
|
}
|
|
ucs |= (*inputOffset++) & 0x3f;
|
|
- if (ucs < 0x80) return SetError (ds, -1, "Overlong 2 byte UTF-8 sequence detected when decoding 'string'");
|
|
+ if (ucs < 0x80) return SetError (ds, -1, "Overlong 2-byte UTF-8 sequence detected when decoding 'string'");
|
|
*(escOffset++) = (JSUINT32) ucs;
|
|
break;
|
|
}
|
|
@@ -554,7 +554,7 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds
|
|
ucs |= oct & 0x3f;
|
|
}
|
|
|
|
- if (ucs < 0x800) return SetError (ds, -1, "Overlong 3 byte UTF-8 sequence detected when encoding string");
|
|
+ if (ucs < 0x800) return SetError (ds, -1, "Overlong 3-byte UTF-8 sequence detected when encoding string");
|
|
*(escOffset++) = (JSUINT32) ucs;
|
|
break;
|
|
}
|
|
@@ -577,7 +577,7 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds
|
|
ucs |= oct & 0x3f;
|
|
}
|
|
|
|
- if (ucs < 0x10000) return SetError (ds, -1, "Overlong 4 byte UTF-8 sequence detected when decoding 'string'");
|
|
+ if (ucs < 0x10000) return SetError (ds, -1, "Overlong 4-byte UTF-8 sequence detected when decoding 'string'");
|
|
|
|
*(escOffset++) = (JSUINT32) ucs;
|
|
break;
|
|
diff --git a/src/ujson/lib/ultrajsonenc.c b/src/ujson/lib/ultrajsonenc.c
|
|
index 0f9fde3f..5bafd522 100644
|
|
--- a/src/ujson/lib/ultrajsonenc.c
|
|
+++ b/src/ujson/lib/ultrajsonenc.c
|
|
@@ -347,17 +347,24 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons
|
|
continue;
|
|
}
|
|
|
|
+ // https://en.wikipedia.org/wiki/UTF-8#Description
|
|
case 2:
|
|
{
|
|
JSUTF32 in;
|
|
JSUTF16 in16;
|
|
|
|
- if (end - io < 1)
|
|
+ if (end - io < 2)
|
|
{
|
|
enc->offset += (of - enc->offset);
|
|
SetError (obj, enc, "Unterminated UTF-8 sequence when encoding string");
|
|
return FALSE;
|
|
}
|
|
+ if ((io[1] & 0xc0) != 0x80)
|
|
+ {
|
|
+ enc->offset += (of - enc->offset);
|
|
+ SetError (obj, enc, "Invalid continuation byte in 2-byte UTF-8 sequence detected when encoding string");
|
|
+ return FALSE;
|
|
+ }
|
|
|
|
memcpy(&in16, io, sizeof(JSUTF16));
|
|
in = (JSUTF32) in16;
|
|
@@ -371,7 +378,7 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons
|
|
if (ucs < 0x80)
|
|
{
|
|
enc->offset += (of - enc->offset);
|
|
- SetError (obj, enc, "Overlong 2 byte UTF-8 sequence detected when encoding string");
|
|
+ SetError (obj, enc, "Overlong 2-byte UTF-8 sequence detected when encoding string");
|
|
return FALSE;
|
|
}
|
|
|
|
@@ -385,13 +392,26 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons
|
|
JSUTF16 in16;
|
|
JSUINT8 in8;
|
|
|
|
- if (end - io < 2)
|
|
+ if (end - io < 3)
|
|
{
|
|
enc->offset += (of - enc->offset);
|
|
SetError (obj, enc, "Unterminated UTF-8 sequence when encoding string");
|
|
return FALSE;
|
|
}
|
|
-
|
|
+ if ((io[1] & 0xc0) != 0x80 || (io[2] & 0xc0) != 0x80)
|
|
+ {
|
|
+ enc->offset += (of - enc->offset);
|
|
+ SetError (obj, enc, "Invalid continuation byte in 3-byte UTF-8 sequence detected when encoding string");
|
|
+ return FALSE;
|
|
+ }
|
|
+ // Under normal UTF-8 decoding rules, UTF-16 surrogates should also be disallowed
|
|
+ // but in JSON, they're special cased and rewritten later as \udc7f.
|
|
+ // if ((JSUINT8) io[0] == 0xed && (JSUINT8) io[1] >= 0xa0)
|
|
+ // {
|
|
+ // enc->offset += (of - enc->offset);
|
|
+ // SetError (obj, enc, "Illegal UTF-16 surrogate in 3-byte UTF-8 sequence detected when encoding string");
|
|
+ // return FALSE;
|
|
+ // }
|
|
memcpy(&in16, io, sizeof(JSUTF16));
|
|
memcpy(&in8, io + 2, sizeof(JSUINT8));
|
|
#ifdef __LITTLE_ENDIAN__
|
|
@@ -407,7 +427,7 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons
|
|
if (ucs < 0x800)
|
|
{
|
|
enc->offset += (of - enc->offset);
|
|
- SetError (obj, enc, "Overlong 3 byte UTF-8 sequence detected when encoding string");
|
|
+ SetError (obj, enc, "Overlong 3-byte UTF-8 sequence detected when encoding string");
|
|
return FALSE;
|
|
}
|
|
|
|
@@ -418,12 +438,24 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons
|
|
{
|
|
JSUTF32 in;
|
|
|
|
- if (end - io < 3)
|
|
+ if (end - io < 4)
|
|
{
|
|
enc->offset += (of - enc->offset);
|
|
SetError (obj, enc, "Unterminated UTF-8 sequence when encoding string");
|
|
return FALSE;
|
|
}
|
|
+ if ((io[1] & 0xc0) != 0x80 || (io[2] & 0xc0) != 0x80 || (io[3] & 0xc0) != 0x80)
|
|
+ {
|
|
+ enc->offset += (of - enc->offset);
|
|
+ SetError (obj, enc, "Invalid continuation byte in 4-byte UTF-8 sequence detected when encoding string");
|
|
+ return FALSE;
|
|
+ }
|
|
+ if (((JSUINT8) io[0] >= 0xf4 && (JSUINT8) io[1] >= 0x90) || (JSUINT8) io[0] >= 0xf5)
|
|
+ {
|
|
+ enc->offset += (of - enc->offset);
|
|
+ SetError (obj, enc, ">U+10FFFF in 4-byte UTF-8 sequence detected when encoding string");
|
|
+ return FALSE;
|
|
+ }
|
|
|
|
memcpy(&in, io, sizeof(JSUTF32));
|
|
#ifdef __LITTLE_ENDIAN__
|
|
@@ -434,7 +466,7 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons
|
|
if (ucs < 0x10000)
|
|
{
|
|
enc->offset += (of - enc->offset);
|
|
- SetError (obj, enc, "Overlong 4 byte UTF-8 sequence detected when encoding string");
|
|
+ SetError (obj, enc, "Overlong 4-byte UTF-8 sequence detected when encoding string");
|
|
return FALSE;
|
|
}
|
|
|
|
diff --git a/tests/test_ujson.py b/tests/test_ujson.py
|
|
index dcf97892..3c9e1a6a 100644
|
|
--- a/tests/test_ujson.py
|
|
+++ b/tests/test_ujson.py
|
|
@@ -1229,6 +1229,81 @@ def test_reject_bytes_nested(value):
|
|
ujson.dumps(value)
|
|
|
|
|
|
+@pytest.mark.parametrize(
|
|
+ "codepoint",
|
|
+ [0x0, 0x7F, 0x80, 0x7FF, 0x800, 0xFFFF, 0x10000, 0x10FFFF],
|
|
+)
|
|
+def test_reject_bytes_false_codepoint_boundaries(codepoint):
|
|
+ char = chr(codepoint)
|
|
+ assert ujson.loads(ujson.dumps(char.encode(), reject_bytes=False)) == char
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize(
|
|
+ "value, error",
|
|
+ [
|
|
+ # Bad start bytes
|
|
+ (b"\xfd", "Unsupported UTF-8 sequence length when encoding string"),
|
|
+ (b"\xfc:", "Unsupported UTF-8 sequence length when encoding string"),
|
|
+ (b"U>\xfb", "Unsupported UTF-8 sequence length when encoding string"),
|
|
+ (b"\\\xf8\x98\t", "Unsupported UTF-8 sequence length when encoding string"),
|
|
+ (b"\x9b", "'utf-8' codec can't decode byte 0x9b in position 1:"),
|
|
+ (b"B\x8a", "'utf-8' codec can't decode byte 0x8a in position 2:"),
|
|
+ # Bad continuation bytes (any non-start byte not matching 0b10xx_xxxx)
|
|
+ (b"\xcf\x13", "Invalid continuation byte in 2-byte UTF-8 sequence"),
|
|
+ (b"\xcfa", "Invalid continuation byte in 2-byte UTF-8 sequence"),
|
|
+ (b"\xd8\xcf\xd3", "Invalid continuation byte in 2-byte UTF-8 sequence"),
|
|
+ (b"\xd2\t\x8b\x84", "Invalid continuation byte in 2-byte UTF-8 sequence"),
|
|
+ (b"\xe2\x17\xce", "Invalid continuation byte in 3-byte UTF-8 sequence"),
|
|
+ (b"\xe2a\x17\xce", "Invalid continuation byte in 3-byte UTF-8 sequence"),
|
|
+ (b"\xe2\x17a", "Invalid continuation byte in 3-byte UTF-8 sequence"),
|
|
+ (b"\xe0\x9c\xc6\xde", "Invalid continuation byte in 3-byte UTF-8 sequence"),
|
|
+ (b"\xf0H\xce\x9b", "Invalid continuation byte in 4-byte UTF-8 sequence"),
|
|
+ (b"\xf0\xce4\x9b", "Invalid continuation byte in 4-byte UTF-8 sequence"),
|
|
+ # Truncated UTF-8 sequences
|
|
+ (b"\xc3", "Unterminated UTF-8 sequence when encoding string"),
|
|
+ (b"\x8c$\xe3", "Unterminated UTF-8 sequence when encoding string"),
|
|
+ (b"\x8c\xe3$", "Unterminated UTF-8 sequence when encoding string"),
|
|
+ (b"=\x8c\xe36", "Unterminated UTF-8 sequence when encoding string"),
|
|
+ (b"\x08\x11\xe3", "Unterminated UTF-8 sequence when encoding string"),
|
|
+ (b"\xf0\x90\x94", "Unterminated UTF-8 sequence when encoding string"),
|
|
+ # Small codepoints using longer byte sequences than they need
|
|
+ (b"\xc0\xa2", "Overlong 2-byte UTF-8 sequence"),
|
|
+ (b"A\xc1\x9c", "Overlong 2-byte UTF-8 sequence"),
|
|
+ (b"\xc1\xbf", "Overlong 2-byte UTF-8 sequence"),
|
|
+ (b"N\xc0\xb4\xb4", "Overlong 2-byte UTF-8 sequence"),
|
|
+ (b"\xe0\x9d\xb3", "Overlong 3-byte UTF-8 sequence"),
|
|
+ (b"E\xe0\x9e\x8b", "Overlong 3-byte UTF-8 sequence"),
|
|
+ (b"\xe0\x9f\xbf", "Overlong 3-byte UTF-8 sequence"),
|
|
+ (b"\xf0\x80\x80\x80", "Overlong 4-byte UTF-8 sequence"),
|
|
+ (b"\xf0\x8f\xbf\xbf", "Overlong 4-byte UTF-8 sequence"),
|
|
+ (b"\xf0\x85\xa7\xbd", "Overlong 4-byte UTF-8 sequence"),
|
|
+ # Codepoints above unicode max
|
|
+ (b"\xf4\x90\x80\x80", r">U\+10FFFF in 4-byte UTF-8 sequence"),
|
|
+ (b"\xf7\x8f\x99\x90", r">U\+10FFFF in 4-byte UTF-8 sequence"),
|
|
+ (b"\xf7\xbf\xbf\xbf", r">U\+10FFFF in 4-byte UTF-8 sequence"),
|
|
+ ],
|
|
+)
|
|
+def test_dump_bytes_invalid_utf8(value, error):
|
|
+ with pytest.raises((OverflowError, UnicodeDecodeError), match=error):
|
|
+ ujson.dumps(bytes(value), reject_bytes=False)
|
|
+
|
|
+
|
|
+def test_dump_bytes_fuzz():
|
|
+ # ujson.dumps(..., reject_bytes=False) should accept or reject the same byte
|
|
+ # sequences as b"...".decode() when unpaired surrogates are allowed
|
|
+ for seed in range(10000):
|
|
+ r = random.Random(seed)
|
|
+ a = r.randbytes(r.randrange(8))
|
|
+ try:
|
|
+ expected = a.decode(errors="surrogatepass")
|
|
+ except UnicodeDecodeError:
|
|
+ with pytest.raises((UnicodeDecodeError, OverflowError)):
|
|
+ ujson.dumps(a, reject_bytes=False)
|
|
+ else:
|
|
+ actual = ujson.loads(ujson.dumps(a, reject_bytes=False))
|
|
+ assert actual == expected, (a, [bin(i) for i in a], actual, expected)
|
|
+
|
|
+
|
|
def test_encode_special_keys():
|
|
data = {None: 0, True: 1, False: 2}
|
|
assert ujson.dumps(data) == '{"null":0,"true":1,"false":2}'
|
|
--
|
|
2.53.0
|
|
|