From eec3f13960d1ee2b6c28c00d9664ab80007f752e Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 26 Mar 2026 10:41:02 +0100 Subject: [PATCH 1/2] feat: support js type overrides for map keys and values Add `jskeytype` and `jsvaluetype` options to support downcasting 64 bit number types to numbers/strings as map key/values. --- .github/dictionary.txt | 2 + packages/protons/README.md | 13 +- packages/protons/src/fields/field.ts | 2 +- packages/protons/src/fields/map-field.ts | 22 +- packages/protons/src/index.ts | 13 +- packages/protons/src/types/message.ts | 22 +- packages/protons/test/custom-options.spec.ts | 8 +- .../test/fixtures/custom-option-jstype.proto | 4 + .../test/fixtures/custom-option-jstype.ts | 426 +++++++++++++++++- 9 files changed, 473 insertions(+), 39 deletions(-) diff --git a/.github/dictionary.txt b/.github/dictionary.txt index 7b2a1d2..8d0781d 100644 --- a/.github/dictionary.txt +++ b/.github/dictionary.txt @@ -10,6 +10,8 @@ deopt transpiles ints jstype +jskeytype +jsmaptype oneofs arraylist bigintify diff --git a/packages/protons/README.md b/packages/protons/README.md index 80ddfc2..5226c1e 100644 --- a/packages/protons/README.md +++ b/packages/protons/README.md @@ -189,13 +189,15 @@ By default 64 bit types are implemented as [BigInt](https://developer.mozilla.or Sometimes this is undesirable due to [performance issues](https://betterprogramming.pub/the-downsides-of-bigints-in-javascript-6350fd807d) or code legibility. -It's possible to override the JavaScript type 64 bit fields will deserialize to: +It's possible to override the JavaScript type 64 bit fields will deserialize +to, including repeated fields and map key/values: ``` message MyMessage { - repeated int64 bigintField = 1; - repeated int64 numberField = 2 [jstype = JS_NUMBER]; - repeated int64 stringField = 3 [jstype = JS_STRING]; + int64 bigintField = 1; + int64 numberField = 2 [jstype = JS_NUMBER]; + repeated int64 stringArray = 3 [jstype = JS_STRING]; + map = { uint64: CODEC_TYPES.VARINT } -const jsTypeOverrides: Record = { +export const jsTypeOverrides: Record = { JS_NUMBER: 'number', JS_STRING: 'string' } diff --git a/packages/protons/src/fields/map-field.ts b/packages/protons/src/fields/map-field.ts index 7b5d797..579189b 100644 --- a/packages/protons/src/fields/map-field.ts +++ b/packages/protons/src/fields/map-field.ts @@ -1,4 +1,4 @@ -import { codecTypes, Field } from './field.ts' +import { codecTypes, Field, jsTypeOverrides } from './field.ts' import type { FieldDef } from './field.ts' import type { Parent } from '../types/index.ts' @@ -15,6 +15,8 @@ export class MapField extends Field { public keyType: string public valueType: string public entryType: string + public jsKeyTypeOverride?: 'string' | 'number' + public jsValueTypeOverride?: 'string' | 'number' private lengthLimit?: number constructor (name: string, def: MapFieldDef, parent: Parent) { @@ -25,14 +27,28 @@ export class MapField extends Field { this.valueType = def.valueType this.entryType = def.type this.lengthLimit = def.options?.['(protons.options).limit'] + + if (def.options?.jskeytype != null) { + this.jsKeyTypeOverride = jsTypeOverrides[def.options.jskeytype] + } + + if (def.options?.jsvaluetype != null) { + this.jsValueTypeOverride = jsTypeOverrides[def.options.jsvaluetype] + } } getInterfaceField (parent: Parent): string { - return `${this.name}: Map<${parent.findType(this.keyType).jsType}, ${parent.findType(this.valueType).jsType}>` + const keyType = this.jsKeyTypeOverride ?? parent.findType(this.keyType).jsType + const valueType = this.jsValueTypeOverride ?? parent.findType(this.valueType).jsType + + return `${this.name}: Map<${keyType}, ${valueType}>` } getDefaultField (parent: Parent): string { - return `${this.name}: new Map<${parent.findType(this.keyType).jsType}, ${parent.findType(this.valueType).jsType}>()` + const keyType = this.jsKeyTypeOverride ?? parent.findType(this.keyType).jsType + const valueType = this.jsValueTypeOverride ?? parent.findType(this.valueType).jsType + + return `${this.name}: new Map<${keyType}, ${valueType}>()` } getEncoder (parent: Parent): string { diff --git a/packages/protons/src/index.ts b/packages/protons/src/index.ts index fd87d21..51af42f 100644 --- a/packages/protons/src/index.ts +++ b/packages/protons/src/index.ts @@ -166,13 +166,15 @@ * * Sometimes this is undesirable due to [performance issues](https://betterprogramming.pub/the-downsides-of-bigints-in-javascript-6350fd807d) or code legibility. * - * It's possible to override the JavaScript type 64 bit fields will deserialize to: + * It's possible to override the JavaScript type 64 bit fields will deserialize + * to, including repeated fields and map key/values: * * ``` * message MyMessage { - * repeated int64 bigintField = 1; - * repeated int64 numberField = 2 [jstype = JS_NUMBER]; - * repeated int64 stringField = 3 [jstype = JS_STRING]; + * int64 bigintField = 1; + * int64 numberField = 2 [jstype = JS_NUMBER]; + * repeated int64 stringArray = 3 [jstype = JS_STRING]; + * map { ui64: 5, si64: 5, f64: 5, - sf64: 5 + sf64: 5, + i64Array: [], + i64Map: new Map() } expect(CustomOptionNumber.decode(CustomOptionNumber.encode(obj))) @@ -23,7 +25,9 @@ describe('custom options', () => { ui64: '5', si64: '5', f64: '5', - sf64: '5' + sf64: '5', + i64Array: ['5'], + i64Map: new Map([['5', '5']]) } expect(CustomOptionString.decode(CustomOptionString.encode(obj))) diff --git a/packages/protons/test/fixtures/custom-option-jstype.proto b/packages/protons/test/fixtures/custom-option-jstype.proto index 777c684..3ff98ac 100644 --- a/packages/protons/test/fixtures/custom-option-jstype.proto +++ b/packages/protons/test/fixtures/custom-option-jstype.proto @@ -7,6 +7,8 @@ message CustomOptionNumber { sint64 si64 = 4 [jstype = JS_NUMBER]; fixed64 f64 = 5 [jstype = JS_NUMBER]; sfixed64 sf64 = 6 [jstype = JS_NUMBER]; + repeated int64 i64Array = 7 [jstype = JS_NUMBER]; + map i64Map = 8 [jskeytype = JS_NUMBER, jsvaluetype = JS_NUMBER]; } message CustomOptionString { @@ -16,4 +18,6 @@ message CustomOptionString { sint64 si64 = 4 [jstype = JS_STRING]; fixed64 f64 = 5 [jstype = JS_STRING]; sfixed64 sf64 = 6 [jstype = JS_STRING]; + repeated int64 i64Array = 7 [jstype = JS_STRING]; + map i64Map = 8 [jskeytype = JS_STRING, jsvaluetype = JS_STRING]; } diff --git a/packages/protons/test/fixtures/custom-option-jstype.ts b/packages/protons/test/fixtures/custom-option-jstype.ts index 78b61ac..d0aebf4 100644 --- a/packages/protons/test/fixtures/custom-option-jstype.ts +++ b/packages/protons/test/fixtures/custom-option-jstype.ts @@ -1,4 +1,4 @@ -import { decodeMessage, encodeMessage, message, streamMessage } from 'protons-runtime' +import { decodeMessage, encodeMessage, MaxLengthError, MaxSizeError, message, streamMessage } from 'protons-runtime' import type { Codec, DecodeOptions } from 'protons-runtime' import type { Uint8ArrayList } from 'uint8arraylist' @@ -9,9 +9,123 @@ export interface CustomOptionNumber { si64: number f64: number sf64: number + i64Array: number[] + i64Map: Map } export namespace CustomOptionNumber { + export interface CustomOptionNumber$i64MapEntry { + key: number + value: number + } + + export namespace CustomOptionNumber$i64MapEntry { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if ((obj.key != null && obj.key !== 0)) { + w.uint32(8) + w.int64Number(obj.key) + } + + if ((obj.value != null && obj.value !== 0)) { + w.uint32(16) + w.int64Number(obj.value) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length, opts = {}) => { + const obj: any = { + key: 0, + value: 0 + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.key = reader.int64Number() + break + } + case 2: { + obj.value = reader.int64Number() + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }, function * (reader, length, prefix, opts = {}) { + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + yield { + field: `${prefix}.key`, + value: reader.int64Number() + } + break + } + case 2: { + yield { + field: `${prefix}.value`, + value: reader.int64Number() + } + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + }) + } + + return _codec + } + + export interface CustomOptionNumber$i64MapEntryKeyFieldEvent { + field: '$.key' + value: number + } + + export interface CustomOptionNumber$i64MapEntryValueFieldEvent { + field: '$.value' + value: number + } + + export function encode (obj: Partial): Uint8Array { + return encodeMessage(obj, CustomOptionNumber$i64MapEntry.codec()) + } + + export function decode (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions): CustomOptionNumber$i64MapEntry { + return decodeMessage(buf, CustomOptionNumber$i64MapEntry.codec(), opts) + } + + export function stream (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions): Generator { + return streamMessage(buf, CustomOptionNumber$i64MapEntry.codec(), opts) + } + } + let _codec: Codec export const codec = (): Codec => { @@ -51,6 +165,20 @@ export namespace CustomOptionNumber { w.sfixed64Number(obj.sf64) } + if (obj.i64Array != null) { + for (const value of obj.i64Array) { + w.uint32(56) + w.int64Number(value) + } + } + + if (obj.i64Map != null) { + for (const [key, value] of obj.i64Map.entries()) { + w.uint32(66) + CustomOptionNumber.CustomOptionNumber$i64MapEntry.codec().encode({ key, value }, w) + } + } + if (opts.lengthDelimited !== false) { w.ldelim() } @@ -61,7 +189,9 @@ export namespace CustomOptionNumber { ui64: 0, si64: 0, f64: 0, - sf64: 0 + sf64: 0, + i64Array: [], + i64Map: new Map() } const end = length == null ? reader.len : reader.pos + length @@ -94,6 +224,27 @@ export namespace CustomOptionNumber { obj.sf64 = reader.sfixed64Number() break } + case 7: { + if (opts.limits?.i64Array != null && obj.i64Array.length === opts.limits.i64Array) { + throw new MaxLengthError('Decode error - repeated field "i64Array" had too many elements') + } + + obj.i64Array.push(reader.int64Number()) + break + } + case 8: { + if (opts.limits?.i64Map != null && obj.i64Map.size === opts.limits.i64Map) { + throw new MaxSizeError('Decode error - map field "i64Map" had too many elements') + } + + const entry = CustomOptionNumber.CustomOptionNumber$i64MapEntry.codec().decode(reader, reader.uint32(), { + limits: { + value: opts.limits?.i64Map$value + } + }) + obj.i64Map.set(entry.key, entry.value) + break + } default: { reader.skipType(tag & 7) break @@ -103,6 +254,11 @@ export namespace CustomOptionNumber { return obj }, function * (reader, length, prefix, opts = {}) { + const obj = { + i64Array: 0, + i64Map: 0 + } + const end = length == null ? reader.len : reader.pos + length while (reader.pos < end) { @@ -151,6 +307,36 @@ export namespace CustomOptionNumber { } break } + case 7: { + if (opts.limits?.i64Array != null && obj.i64Array === opts.limits.i64Array) { + throw new MaxLengthError('Streaming decode error - repeated field "i64Array" had too many elements') + } + + yield { + field: `${prefix}.i64Array[]`, + index: obj.i64Array, + value: reader.int64Number() + } + + obj.i64Array++ + + break + } + case 8: { + if (opts.limits?.i64Map != null && obj.i64Map === opts.limits.i64Map) { + throw new MaxLengthError('Decode error - map field "i64Map" had too many elements') + } + + yield * CustomOptionNumber.CustomOptionNumber$i64MapEntry.codec().stream(reader, reader.uint32(), `${prefix}.i64Map{}`, { + limits: { + value: opts.limits?.i64Map$value + } + }) + + obj.i64Map++ + + break + } default: { reader.skipType(tag & 7) break @@ -170,27 +356,39 @@ export namespace CustomOptionNumber { export interface CustomOptionNumberI64FieldEvent { field: '$.i64' - value: bigint + value: number } export interface CustomOptionNumberUi64FieldEvent { field: '$.ui64' - value: bigint + value: number } export interface CustomOptionNumberSi64FieldEvent { field: '$.si64' - value: bigint + value: number } export interface CustomOptionNumberF64FieldEvent { field: '$.f64' - value: bigint + value: number } export interface CustomOptionNumberSf64FieldEvent { field: '$.sf64' - value: bigint + value: number + } + + export interface CustomOptionNumberI64ArrayFieldEvent { + field: '$.i64Array[]' + index: number + value: number + } + + export interface CustomOptionNumberI64MapFieldEvent { + field: '$.i64Map{}' + key: number + value: number } export function encode (obj: Partial): Uint8Array { @@ -201,7 +399,7 @@ export namespace CustomOptionNumber { return decodeMessage(buf, CustomOptionNumber.codec(), opts) } - export function stream (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions): Generator { + export function stream (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions): Generator { return streamMessage(buf, CustomOptionNumber.codec(), opts) } } @@ -213,9 +411,123 @@ export interface CustomOptionString { si64: string f64: string sf64: string + i64Array: string[] + i64Map: Map } export namespace CustomOptionString { + export interface CustomOptionString$i64MapEntry { + key: string + value: string + } + + export namespace CustomOptionString$i64MapEntry { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if ((obj.key != null && obj.key !== '')) { + w.uint32(8) + w.int64String(obj.key) + } + + if ((obj.value != null && obj.value !== '')) { + w.uint32(16) + w.int64String(obj.value) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length, opts = {}) => { + const obj: any = { + key: '', + value: '' + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.key = reader.int64String() + break + } + case 2: { + obj.value = reader.int64String() + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }, function * (reader, length, prefix, opts = {}) { + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + yield { + field: `${prefix}.key`, + value: reader.int64String() + } + break + } + case 2: { + yield { + field: `${prefix}.value`, + value: reader.int64String() + } + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + }) + } + + return _codec + } + + export interface CustomOptionString$i64MapEntryKeyFieldEvent { + field: '$.key' + value: string + } + + export interface CustomOptionString$i64MapEntryValueFieldEvent { + field: '$.value' + value: string + } + + export function encode (obj: Partial): Uint8Array { + return encodeMessage(obj, CustomOptionString$i64MapEntry.codec()) + } + + export function decode (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions): CustomOptionString$i64MapEntry { + return decodeMessage(buf, CustomOptionString$i64MapEntry.codec(), opts) + } + + export function stream (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions): Generator { + return streamMessage(buf, CustomOptionString$i64MapEntry.codec(), opts) + } + } + let _codec: Codec export const codec = (): Codec => { @@ -255,6 +567,20 @@ export namespace CustomOptionString { w.sfixed64String(obj.sf64) } + if (obj.i64Array != null) { + for (const value of obj.i64Array) { + w.uint32(56) + w.int64String(value) + } + } + + if (obj.i64Map != null) { + for (const [key, value] of obj.i64Map.entries()) { + w.uint32(66) + CustomOptionString.CustomOptionString$i64MapEntry.codec().encode({ key, value }, w) + } + } + if (opts.lengthDelimited !== false) { w.ldelim() } @@ -265,7 +591,9 @@ export namespace CustomOptionString { ui64: '', si64: '', f64: '', - sf64: '' + sf64: '', + i64Array: [], + i64Map: new Map() } const end = length == null ? reader.len : reader.pos + length @@ -298,6 +626,27 @@ export namespace CustomOptionString { obj.sf64 = reader.sfixed64String() break } + case 7: { + if (opts.limits?.i64Array != null && obj.i64Array.length === opts.limits.i64Array) { + throw new MaxLengthError('Decode error - repeated field "i64Array" had too many elements') + } + + obj.i64Array.push(reader.int64String()) + break + } + case 8: { + if (opts.limits?.i64Map != null && obj.i64Map.size === opts.limits.i64Map) { + throw new MaxSizeError('Decode error - map field "i64Map" had too many elements') + } + + const entry = CustomOptionString.CustomOptionString$i64MapEntry.codec().decode(reader, reader.uint32(), { + limits: { + value: opts.limits?.i64Map$value + } + }) + obj.i64Map.set(entry.key, entry.value) + break + } default: { reader.skipType(tag & 7) break @@ -307,6 +656,11 @@ export namespace CustomOptionString { return obj }, function * (reader, length, prefix, opts = {}) { + const obj = { + i64Array: 0, + i64Map: 0 + } + const end = length == null ? reader.len : reader.pos + length while (reader.pos < end) { @@ -355,6 +709,36 @@ export namespace CustomOptionString { } break } + case 7: { + if (opts.limits?.i64Array != null && obj.i64Array === opts.limits.i64Array) { + throw new MaxLengthError('Streaming decode error - repeated field "i64Array" had too many elements') + } + + yield { + field: `${prefix}.i64Array[]`, + index: obj.i64Array, + value: reader.int64String() + } + + obj.i64Array++ + + break + } + case 8: { + if (opts.limits?.i64Map != null && obj.i64Map === opts.limits.i64Map) { + throw new MaxLengthError('Decode error - map field "i64Map" had too many elements') + } + + yield * CustomOptionString.CustomOptionString$i64MapEntry.codec().stream(reader, reader.uint32(), `${prefix}.i64Map{}`, { + limits: { + value: opts.limits?.i64Map$value + } + }) + + obj.i64Map++ + + break + } default: { reader.skipType(tag & 7) break @@ -374,27 +758,39 @@ export namespace CustomOptionString { export interface CustomOptionStringI64FieldEvent { field: '$.i64' - value: bigint + value: string } export interface CustomOptionStringUi64FieldEvent { field: '$.ui64' - value: bigint + value: string } export interface CustomOptionStringSi64FieldEvent { field: '$.si64' - value: bigint + value: string } export interface CustomOptionStringF64FieldEvent { field: '$.f64' - value: bigint + value: string } export interface CustomOptionStringSf64FieldEvent { field: '$.sf64' - value: bigint + value: string + } + + export interface CustomOptionStringI64ArrayFieldEvent { + field: '$.i64Array[]' + index: number + value: string + } + + export interface CustomOptionStringI64MapFieldEvent { + field: '$.i64Map{}' + key: string + value: string } export function encode (obj: Partial): Uint8Array { @@ -405,7 +801,7 @@ export namespace CustomOptionString { return decodeMessage(buf, CustomOptionString.codec(), opts) } - export function stream (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions): Generator { + export function stream (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions): Generator { return streamMessage(buf, CustomOptionString.codec(), opts) } } From 10c51e4aa9f05f740b02c35582cda0bd16929fb3 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 26 Mar 2026 10:48:33 +0100 Subject: [PATCH 2/2] chore: spelling --- .github/dictionary.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dictionary.txt b/.github/dictionary.txt index 8d0781d..4642079 100644 --- a/.github/dictionary.txt +++ b/.github/dictionary.txt @@ -11,7 +11,7 @@ transpiles ints jstype jskeytype -jsmaptype +jsvaluetype oneofs arraylist bigintify