From 169eaf36b1116fece5034ee79a7a0ef3f6deedcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Br=C3=A9nainn=20Woodsend?= Date: Fri, 24 Apr 2026 21:59:45 +0100 Subject: [PATCH] More UTF-8 validation for ujson.dumps(b"...", reject_bytes=False) * Fix off by one errors in detecting end of string mid sequence * Add missing check for codepoints > max unicode * Add missing check for bad continuation bytes CVE: CVE-2026-54911 Upstream-Status: Backport [https://github.com/ultrajson/ultrajson/commit/169eaf36b1116fece5034ee79a7a0ef3f6deedcf] (cherry picked from commit 169eaf36b1116fece5034ee79a7a0ef3f6deedcf) Signed-off-by: Hetvi Thakar --- src/ujson/lib/ultrajsondec.c | 6 +-- src/ujson/lib/ultrajsonenc.c | 46 ++++++++++++++++++---- tests/test_ujson.py | 75 ++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 10 deletions(-) diff --git a/src/ujson/lib/ultrajsondec.c b/src/ujson/lib/ultrajsondec.c index bccb0aaf..55833763 100644 --- a/src/ujson/lib/ultrajsondec.c +++ b/src/ujson/lib/ultrajsondec.c @@ -531,7 +531,7 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds return SetError(ds, -1, "Invalid octet in UTF-8 sequence when decoding 'string'"); } ucs |= (*inputOffset++) & 0x3f; - if (ucs < 0x80) return SetError (ds, -1, "Overlong 2 byte UTF-8 sequence detected when decoding 'string'"); + if (ucs < 0x80) return SetError (ds, -1, "Overlong 2-byte UTF-8 sequence detected when decoding 'string'"); *(escOffset++) = (JSUINT32) ucs; break; } @@ -554,7 +554,7 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds ucs |= oct & 0x3f; } - if (ucs < 0x800) return SetError (ds, -1, "Overlong 3 byte UTF-8 sequence detected when encoding string"); + if (ucs < 0x800) return SetError (ds, -1, "Overlong 3-byte UTF-8 sequence detected when encoding string"); *(escOffset++) = (JSUINT32) ucs; break; } @@ -577,7 +577,7 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds ucs |= oct & 0x3f; } - if (ucs < 0x10000) return SetError (ds, -1, "Overlong 4 byte UTF-8 sequence detected when decoding 'string'"); + if (ucs < 0x10000) return SetError (ds, -1, "Overlong 4-byte UTF-8 sequence detected when decoding 'string'"); *(escOffset++) = (JSUINT32) ucs; break; diff --git a/src/ujson/lib/ultrajsonenc.c b/src/ujson/lib/ultrajsonenc.c index 0f9fde3f..5bafd522 100644 --- a/src/ujson/lib/ultrajsonenc.c +++ b/src/ujson/lib/ultrajsonenc.c @@ -347,17 +347,24 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons continue; } + // https://en.wikipedia.org/wiki/UTF-8#Description case 2: { JSUTF32 in; JSUTF16 in16; - if (end - io < 1) + if (end - io < 2) { enc->offset += (of - enc->offset); SetError (obj, enc, "Unterminated UTF-8 sequence when encoding string"); return FALSE; } + if ((io[1] & 0xc0) != 0x80) + { + enc->offset += (of - enc->offset); + SetError (obj, enc, "Invalid continuation byte in 2-byte UTF-8 sequence detected when encoding string"); + return FALSE; + } memcpy(&in16, io, sizeof(JSUTF16)); in = (JSUTF32) in16; @@ -371,7 +378,7 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons if (ucs < 0x80) { enc->offset += (of - enc->offset); - SetError (obj, enc, "Overlong 2 byte UTF-8 sequence detected when encoding string"); + SetError (obj, enc, "Overlong 2-byte UTF-8 sequence detected when encoding string"); return FALSE; } @@ -385,13 +392,26 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons JSUTF16 in16; JSUINT8 in8; - if (end - io < 2) + if (end - io < 3) { enc->offset += (of - enc->offset); SetError (obj, enc, "Unterminated UTF-8 sequence when encoding string"); return FALSE; } - + if ((io[1] & 0xc0) != 0x80 || (io[2] & 0xc0) != 0x80) + { + enc->offset += (of - enc->offset); + SetError (obj, enc, "Invalid continuation byte in 3-byte UTF-8 sequence detected when encoding string"); + return FALSE; + } + // Under normal UTF-8 decoding rules, UTF-16 surrogates should also be disallowed + // but in JSON, they're special cased and rewritten later as \udc7f. + // if ((JSUINT8) io[0] == 0xed && (JSUINT8) io[1] >= 0xa0) + // { + // enc->offset += (of - enc->offset); + // SetError (obj, enc, "Illegal UTF-16 surrogate in 3-byte UTF-8 sequence detected when encoding string"); + // return FALSE; + // } memcpy(&in16, io, sizeof(JSUTF16)); memcpy(&in8, io + 2, sizeof(JSUINT8)); #ifdef __LITTLE_ENDIAN__ @@ -407,7 +427,7 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons if (ucs < 0x800) { enc->offset += (of - enc->offset); - SetError (obj, enc, "Overlong 3 byte UTF-8 sequence detected when encoding string"); + SetError (obj, enc, "Overlong 3-byte UTF-8 sequence detected when encoding string"); return FALSE; } @@ -418,12 +438,24 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons { JSUTF32 in; - if (end - io < 3) + if (end - io < 4) { enc->offset += (of - enc->offset); SetError (obj, enc, "Unterminated UTF-8 sequence when encoding string"); return FALSE; } + if ((io[1] & 0xc0) != 0x80 || (io[2] & 0xc0) != 0x80 || (io[3] & 0xc0) != 0x80) + { + enc->offset += (of - enc->offset); + SetError (obj, enc, "Invalid continuation byte in 4-byte UTF-8 sequence detected when encoding string"); + return FALSE; + } + if (((JSUINT8) io[0] >= 0xf4 && (JSUINT8) io[1] >= 0x90) || (JSUINT8) io[0] >= 0xf5) + { + enc->offset += (of - enc->offset); + SetError (obj, enc, ">U+10FFFF in 4-byte UTF-8 sequence detected when encoding string"); + return FALSE; + } memcpy(&in, io, sizeof(JSUTF32)); #ifdef __LITTLE_ENDIAN__ @@ -434,7 +466,7 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons if (ucs < 0x10000) { enc->offset += (of - enc->offset); - SetError (obj, enc, "Overlong 4 byte UTF-8 sequence detected when encoding string"); + SetError (obj, enc, "Overlong 4-byte UTF-8 sequence detected when encoding string"); return FALSE; } diff --git a/tests/test_ujson.py b/tests/test_ujson.py index dcf97892..3c9e1a6a 100644 --- a/tests/test_ujson.py +++ b/tests/test_ujson.py @@ -1229,6 +1229,81 @@ def test_reject_bytes_nested(value): ujson.dumps(value) +@pytest.mark.parametrize( + "codepoint", + [0x0, 0x7F, 0x80, 0x7FF, 0x800, 0xFFFF, 0x10000, 0x10FFFF], +) +def test_reject_bytes_false_codepoint_boundaries(codepoint): + char = chr(codepoint) + assert ujson.loads(ujson.dumps(char.encode(), reject_bytes=False)) == char + + +@pytest.mark.parametrize( + "value, error", + [ + # Bad start bytes + (b"\xfd", "Unsupported UTF-8 sequence length when encoding string"), + (b"\xfc:", "Unsupported UTF-8 sequence length when encoding string"), + (b"U>\xfb", "Unsupported UTF-8 sequence length when encoding string"), + (b"\\\xf8\x98\t", "Unsupported UTF-8 sequence length when encoding string"), + (b"\x9b", "'utf-8' codec can't decode byte 0x9b in position 1:"), + (b"B\x8a", "'utf-8' codec can't decode byte 0x8a in position 2:"), + # Bad continuation bytes (any non-start byte not matching 0b10xx_xxxx) + (b"\xcf\x13", "Invalid continuation byte in 2-byte UTF-8 sequence"), + (b"\xcfa", "Invalid continuation byte in 2-byte UTF-8 sequence"), + (b"\xd8\xcf\xd3", "Invalid continuation byte in 2-byte UTF-8 sequence"), + (b"\xd2\t\x8b\x84", "Invalid continuation byte in 2-byte UTF-8 sequence"), + (b"\xe2\x17\xce", "Invalid continuation byte in 3-byte UTF-8 sequence"), + (b"\xe2a\x17\xce", "Invalid continuation byte in 3-byte UTF-8 sequence"), + (b"\xe2\x17a", "Invalid continuation byte in 3-byte UTF-8 sequence"), + (b"\xe0\x9c\xc6\xde", "Invalid continuation byte in 3-byte UTF-8 sequence"), + (b"\xf0H\xce\x9b", "Invalid continuation byte in 4-byte UTF-8 sequence"), + (b"\xf0\xce4\x9b", "Invalid continuation byte in 4-byte UTF-8 sequence"), + # Truncated UTF-8 sequences + (b"\xc3", "Unterminated UTF-8 sequence when encoding string"), + (b"\x8c$\xe3", "Unterminated UTF-8 sequence when encoding string"), + (b"\x8c\xe3$", "Unterminated UTF-8 sequence when encoding string"), + (b"=\x8c\xe36", "Unterminated UTF-8 sequence when encoding string"), + (b"\x08\x11\xe3", "Unterminated UTF-8 sequence when encoding string"), + (b"\xf0\x90\x94", "Unterminated UTF-8 sequence when encoding string"), + # Small codepoints using longer byte sequences than they need + (b"\xc0\xa2", "Overlong 2-byte UTF-8 sequence"), + (b"A\xc1\x9c", "Overlong 2-byte UTF-8 sequence"), + (b"\xc1\xbf", "Overlong 2-byte UTF-8 sequence"), + (b"N\xc0\xb4\xb4", "Overlong 2-byte UTF-8 sequence"), + (b"\xe0\x9d\xb3", "Overlong 3-byte UTF-8 sequence"), + (b"E\xe0\x9e\x8b", "Overlong 3-byte UTF-8 sequence"), + (b"\xe0\x9f\xbf", "Overlong 3-byte UTF-8 sequence"), + (b"\xf0\x80\x80\x80", "Overlong 4-byte UTF-8 sequence"), + (b"\xf0\x8f\xbf\xbf", "Overlong 4-byte UTF-8 sequence"), + (b"\xf0\x85\xa7\xbd", "Overlong 4-byte UTF-8 sequence"), + # Codepoints above unicode max + (b"\xf4\x90\x80\x80", r">U\+10FFFF in 4-byte UTF-8 sequence"), + (b"\xf7\x8f\x99\x90", r">U\+10FFFF in 4-byte UTF-8 sequence"), + (b"\xf7\xbf\xbf\xbf", r">U\+10FFFF in 4-byte UTF-8 sequence"), + ], +) +def test_dump_bytes_invalid_utf8(value, error): + with pytest.raises((OverflowError, UnicodeDecodeError), match=error): + ujson.dumps(bytes(value), reject_bytes=False) + + +def test_dump_bytes_fuzz(): + # ujson.dumps(..., reject_bytes=False) should accept or reject the same byte + # sequences as b"...".decode() when unpaired surrogates are allowed + for seed in range(10000): + r = random.Random(seed) + a = r.randbytes(r.randrange(8)) + try: + expected = a.decode(errors="surrogatepass") + except UnicodeDecodeError: + with pytest.raises((UnicodeDecodeError, OverflowError)): + ujson.dumps(a, reject_bytes=False) + else: + actual = ujson.loads(ujson.dumps(a, reject_bytes=False)) + assert actual == expected, (a, [bin(i) for i in a], actual, expected) + + def test_encode_special_keys(): data = {None: 0, True: 1, False: 2} assert ujson.dumps(data) == '{"null":0,"true":1,"false":2}' -- 2.53.0