From 9a4703acbfe970b7896c586a6a6c6f31debfe95a Mon Sep 17 00:00:00 2001 From: Keichi Takahashi Date: Sun, 18 Oct 2026 22:46:45 +0900 Subject: [PATCH 1/2] Decode latin1/ascii/base64/base64url strings with Buffer semantics --- README.md | 5 +++-- lib/binary_parser.ts | 41 ++++++++++++++++++++++------------------ test/primitive_parser.ts | 17 +++++++++++++++++ 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 23aa972..8c65423 100644 --- a/README.md +++ b/README.md @@ -149,8 +149,9 @@ characters and start with an alphabet. `options` is an object which can have the following keys: - `encoding` - (Optional, defaults to `utf8`) Specify which encoding to use. - Supported encodings include `"hex"` and all encodings supported by - [`TextDecoder`](https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder/encoding). + Supported encodings are `"utf8"` (`"utf-8"`), `"ascii"`, `"latin1"` + (`"binary"`), `"hex"`, `"base64"` and `"base64url"`, with the same semantics + as Node.js `Buffer#toString()`. - `length ` - (Optional) Length of the string. Can be a number, string or a function. Use number for statically sized arrays, string to reference another variable and function to do some calculation. diff --git a/lib/binary_parser.ts b/lib/binary_parser.ts index cf511f6..318c93c 100644 --- a/lib/binary_parser.ts +++ b/lib/binary_parser.ts @@ -1195,8 +1195,25 @@ export class Parser { const name = ctx.generateVariable(this.varName); const start = ctx.generateTmpVariable(); const encoding = this.options.encoding!; - const isHex = encoding.toLowerCase() === "hex"; - const toHex = 'b => b.toString(16).padStart(2, "0")'; + const latin1 = (bytes: string, mask = "b") => + `Array.from(${bytes}, b => String.fromCharCode(${mask})).join('')`; + const decode = (bytes: string) => { + switch (encoding.toLowerCase()) { + case "hex": + return `Array.from(${bytes}, b => b.toString(16).padStart(2, "0")).join('')`; + case "latin1": + case "binary": + return latin1(bytes); + case "ascii": + return latin1(bytes, "b & 0x7f"); + case "base64": + return `btoa(${latin1(bytes)})`; + case "base64url": + return `btoa(${latin1(bytes)}).replace(/\\+/g, '-').replace(/\\//g, '_').replace(/=+$/, '')`; + default: + return `new TextDecoder('${encoding}').decode(${bytes})`; + } + }; if (this.options.length && this.options.zeroTerminated) { const len = this.options.length; @@ -1205,35 +1222,23 @@ export class Parser { `while(dataView.getUint8(offset++) !== 0 && offset - ${start} < ${len});`, ); const end = `offset - ${start} < ${len} ? offset - 1 : offset`; - ctx.pushCode( - isHex - ? `${name} = Array.from(buffer.subarray(${start}, ${end}), ${toHex}).join('');` - : `${name} = new TextDecoder('${encoding}').decode(buffer.subarray(${start}, ${end}));`, - ); + ctx.pushCode(`${name} = ${decode(`buffer.subarray(${start}, ${end})`)};`); } else if (this.options.length) { const len = ctx.generateOption(this.options.length); ctx.pushCode( - isHex - ? `${name} = Array.from(buffer.subarray(offset, offset + ${len}), ${toHex}).join('');` - : `${name} = new TextDecoder('${encoding}').decode(buffer.subarray(offset, offset + ${len}));`, + `${name} = ${decode(`buffer.subarray(offset, offset + ${len})`)};`, ); ctx.pushCode(`offset += ${len};`); } else if (this.options.zeroTerminated) { ctx.pushCode(`var ${start} = offset;`); ctx.pushCode("while(dataView.getUint8(offset++) !== 0);"); ctx.pushCode( - isHex - ? `${name} = Array.from(buffer.subarray(${start}, offset - 1), ${toHex}).join('');` - : `${name} = new TextDecoder('${encoding}').decode(buffer.subarray(${start}, offset - 1));`, + `${name} = ${decode(`buffer.subarray(${start}, offset - 1)`)};`, ); } else if (this.options.greedy) { ctx.pushCode(`var ${start} = offset;`); ctx.pushCode("while(buffer.length > offset++);"); - ctx.pushCode( - isHex - ? `${name} = Array.from(buffer.subarray(${start}, offset), ${toHex}).join('');` - : `${name} = new TextDecoder('${encoding}').decode(buffer.subarray(${start}, offset));`, - ); + ctx.pushCode(`${name} = ${decode(`buffer.subarray(${start}, offset)`)};`); } if (this.options.stripNull) { ctx.pushCode(`${name} = ${name}.replace(/\\x00+$/g, '')`); diff --git a/test/primitive_parser.ts b/test/primitive_parser.ts index dec80c1..830a312 100644 --- a/test/primitive_parser.ts +++ b/test/primitive_parser.ts @@ -325,6 +325,23 @@ function primitiveParserTests( expect(parser.parse(buffer).msg).toStrictEqual(text); }); + it("should parse latin1, ascii and base64 encoded strings", () => { + const buffer = hexToBuf("4180fffbef"); + const expected = { + latin1: "A\x80\xff\xfb\xef", + binary: "A\x80\xff\xfb\xef", + ascii: "A\x00\x7f\x7b\x6f", + base64: "QYD/++8=", + base64url: "QYD_--8", + }; + for (const [encoding, msg] of Object.entries(expected)) { + const parser = Parser.start().string("msg", { + length: buffer.length, + encoding, + }); + expect(parser.parse(buffer).msg).toStrictEqual(msg); + } + }); it("should parse variable length string", () => { const buffer = hexToBuf("0c68656c6c6f2c20776f726c64"); const parser = Parser.start() From 884dfbbd44ea8a9f9b0cf6aa51907882146efade Mon Sep 17 00:00:00 2001 From: Keichi Takahashi Date: Sun, 18 Oct 2026 23:06:46 +0900 Subject: [PATCH 2/2] Make large string encoding test browser-compatible --- test/primitive_parser.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/primitive_parser.ts b/test/primitive_parser.ts index efe0eed..8ff48b6 100644 --- a/test/primitive_parser.ts +++ b/test/primitive_parser.ts @@ -370,14 +370,20 @@ function primitiveParserTests( it("should parse latin1, ascii and base64 strings larger than a chunk", () => { const bytes = Uint8Array.from({ length: 20000 }, (_, i) => i * 7); const buffer = factory(bytes); - for (const encoding of ["latin1", "ascii", "base64"] as const) { + // Naive per-byte reference (no Buffer, so it also runs in browsers) + let latin1 = ""; + let ascii = ""; + for (const b of bytes) { + latin1 += String.fromCharCode(b); + ascii += String.fromCharCode(b & 0x7f); + } + const expected = { latin1, ascii, base64: btoa(latin1) }; + for (const [encoding, msg] of Object.entries(expected)) { const parser = Parser.start().string("msg", { length: buffer.length, encoding, }); - expect(parser.parse(buffer).msg).toStrictEqual( - Buffer.from(bytes).toString(encoding), - ); + expect(parser.parse(buffer).msg).toStrictEqual(msg); } }); it("should parse variable length string", () => {