diff --git a/README.md b/README.md index 23aa972..8c65423 100644 --- a/README.md +++ b/README.md @@ -149,8 +149,9 @@ characters and start with an alphabet. `options` is an object which can have the following keys: - `encoding` - (Optional, defaults to `utf8`) Specify which encoding to use. - Supported encodings include `"hex"` and all encodings supported by - [`TextDecoder`](https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder/encoding). + Supported encodings are `"utf8"` (`"utf-8"`), `"ascii"`, `"latin1"` + (`"binary"`), `"hex"`, `"base64"` and `"base64url"`, with the same semantics + as Node.js `Buffer#toString()`. - `length ` - (Optional) Length of the string. Can be a number, string or a function. Use number for statically sized arrays, string to reference another variable and function to do some calculation. diff --git a/lib/binary_parser.ts b/lib/binary_parser.ts index 25d54e9..e236880 100644 --- a/lib/binary_parser.ts +++ b/lib/binary_parser.ts @@ -274,6 +274,26 @@ const PRIMITIVE_LITTLE_ENDIANS: { [key in PrimitiveTypes]: boolean } = { doublebe: false, }; +// These helpers must be self-contained (no outer references) since they may +// be inlined via toString(). +function bytesToLatin1(bytes: Uint8Array): string { + let s = ""; + for (let i = 0; i < bytes.length; i += 8192) { + s += String.fromCharCode.apply(null, bytes.subarray(i, i + 8192) as any); + } + return s; +} + +// Like Node.js, unset the high bit; the result is pure ASCII, so UTF-8 decoding +// is exact and much faster than String.fromCharCode. +function bytesToAscii(bytes: Uint8Array): string { + const masked = new Uint8Array(bytes.length); + for (let i = 0; i < bytes.length; i++) { + masked[i] = bytes[i] & 0x7f; + } + return new TextDecoder().decode(masked); +} + export class Parser { varName = ""; type: Types = ""; @@ -1216,8 +1236,25 @@ export class Parser { const name = ctx.generateVariable(this.varName); const start = ctx.generateTmpVariable(); const encoding = this.options.encoding!; - const isHex = encoding.toLowerCase() === "hex"; - const toHex = 'b => b.toString(16).padStart(2, "0")'; + const latin1 = (bytes: string) => + `${ctx.addImport(bytesToLatin1)}(${bytes})`; + const decode = (bytes: string) => { + switch (encoding.toLowerCase()) { + case "hex": + return `Array.from(${bytes}, b => b.toString(16).padStart(2, "0")).join('')`; + case "latin1": + case "binary": + return latin1(bytes); + case "ascii": + return `${ctx.addImport(bytesToAscii)}(${bytes})`; + case "base64": + return `btoa(${latin1(bytes)})`; + case "base64url": + return `btoa(${latin1(bytes)}).replace(/\\+/g, '-').replace(/\\//g, '_').replace(/=+$/, '')`; + default: + return `new TextDecoder('${encoding}').decode(${bytes})`; + } + }; if (this.options.length != null && this.options.zeroTerminated) { const len = ctx.generateTmpVariable(); @@ -1226,36 +1263,24 @@ export class Parser { ctx.pushCode( `while(offset - ${start} < ${len} && dataView.getUint8(offset) !== 0) offset++;`, ); - ctx.pushCode( - isHex - ? `${name} = Array.from(buffer.subarray(${start}, offset), ${toHex}).join('');` - : `${name} = new TextDecoder('${encoding}').decode(buffer.subarray(${start}, offset));`, - ); + ctx.pushCode(`${name} = ${decode(`buffer.subarray(${start}, offset)`)};`); ctx.pushCode(`if (offset - ${start} < ${len}) offset++;`); } else if (this.options.length != null) { const len = ctx.generateOption(this.options.length); ctx.pushCode( - isHex - ? `${name} = Array.from(buffer.subarray(offset, offset + ${len}), ${toHex}).join('');` - : `${name} = new TextDecoder('${encoding}').decode(buffer.subarray(offset, offset + ${len}));`, + `${name} = ${decode(`buffer.subarray(offset, offset + ${len})`)};`, ); ctx.pushCode(`offset += ${len};`); } else if (this.options.zeroTerminated) { ctx.pushCode(`var ${start} = offset;`); ctx.pushCode("while(dataView.getUint8(offset++) !== 0);"); ctx.pushCode( - isHex - ? `${name} = Array.from(buffer.subarray(${start}, offset - 1), ${toHex}).join('');` - : `${name} = new TextDecoder('${encoding}').decode(buffer.subarray(${start}, offset - 1));`, + `${name} = ${decode(`buffer.subarray(${start}, offset - 1)`)};`, ); } else if (this.options.greedy) { ctx.pushCode(`var ${start} = offset;`); ctx.pushCode("offset = buffer.length;"); - ctx.pushCode( - isHex - ? `${name} = Array.from(buffer.subarray(${start}, offset), ${toHex}).join('');` - : `${name} = new TextDecoder('${encoding}').decode(buffer.subarray(${start}, offset));`, - ); + ctx.pushCode(`${name} = ${decode(`buffer.subarray(${start}, offset)`)};`); } if (this.options.stripNull) { ctx.pushCode(`${name} = ${name}.replace(/\\x00+$/g, '')`); diff --git a/test/primitive_parser.ts b/test/primitive_parser.ts index 58f303e..cf653ae 100644 --- a/test/primitive_parser.ts +++ b/test/primitive_parser.ts @@ -350,6 +350,42 @@ function primitiveParserTests( expect(parser.parse(buffer).msg).toStrictEqual(text); }); + it("should parse latin1, ascii and base64 encoded strings", () => { + const buffer = hexToBuf("4180fffbef"); + const expected = { + latin1: "A\x80\xff\xfb\xef", + binary: "A\x80\xff\xfb\xef", + ascii: "A\x00\x7f\x7b\x6f", + base64: "QYD/++8=", + base64url: "QYD_--8", + }; + for (const [encoding, msg] of Object.entries(expected)) { + const parser = Parser.start().string("msg", { + length: buffer.length, + encoding, + }); + expect(parser.parse(buffer).msg).toStrictEqual(msg); + } + }); + it("should parse latin1, ascii and base64 strings larger than a chunk", () => { + const bytes = Uint8Array.from({ length: 20000 }, (_, i) => i * 7); + const buffer = factory(bytes); + // Naive per-byte reference (no Buffer, so it also runs in browsers) + let latin1 = ""; + let ascii = ""; + for (const b of bytes) { + latin1 += String.fromCharCode(b); + ascii += String.fromCharCode(b & 0x7f); + } + const expected = { latin1, ascii, base64: btoa(latin1) }; + for (const [encoding, msg] of Object.entries(expected)) { + const parser = Parser.start().string("msg", { + length: buffer.length, + encoding, + }); + expect(parser.parse(buffer).msg).toStrictEqual(msg); + } + }); it("should parse variable length string", () => { const buffer = hexToBuf("0c68656c6c6f2c20776f726c64"); const parser = Parser.start()