Skip to content

Commit 3e0379c

Browse files
streamichstreamich
committed
fix: 🐛 improve JSON codegen
1 parent 9cba298 commit 3e0379c

File tree

8 files changed

+192
-22
lines changed

8 files changed

+192
-22
lines changed

‎src/__tests__/fixtures.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ export const compositeSchemas = {
6363
s.Array(s.Number()),
6464
),
6565
binary: s.bin,
66+
doubleMap: s.Map(s.Map(s.str)),
67+
nestedMaps: s.Map(s.Map(s.Map(s.nil))),
68+
nestedArrays: s.Array(s.Array(s.Array(s.str))),
6669
} as const;
6770

6871
/**

‎src/codegen/binary/AbstractBinaryCodegen.ts‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,8 @@ import type {
88
BinType,
99
BoolType,
1010
ConType,
11-
KeyType,
1211
MapType,
1312
NumType,
14-
ObjType,
1513
OrType,
1614
RefType,
1715
StrType,
@@ -148,7 +146,21 @@ var uint8 = writer.uint8, view = writer.view;`,
148146
}
149147

150148
protected onBool(path: SchemaPath, r: JsExpression, type: BoolType): void {
151-
this.codegen.js(/* js */ `encoder.writeBoolean(${r.use()});`);
149+
this.codegen.if(`${r.use()}`,
150+
() => {
151+
this.blob(
152+
this.gen((encoder) => {
153+
encoder.writeBoolean(true);
154+
}),
155+
);
156+
},
157+
() => {
158+
this.blob(
159+
this.gen((encoder) => {
160+
encoder.writeBoolean(false);
161+
}),
162+
);
163+
});
152164
}
153165

154166
protected onNum(path: SchemaPath, r: JsExpression, type: NumType): void {

‎src/codegen/binary/json/JsonCodegen.ts‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ export class JsonCodegen extends AbstractBinaryCodegen<JsonEncoder> {
1313
const r = codegen.codegen.options.args[0];
1414
const expression = new JsExpression(() => r);
1515
codegen.onNode([], expression, type);
16+
17+
// console.log(codegen.codegen.generate().js);
1618
return codegen.compile();
1719
});
1820

@@ -142,26 +144,26 @@ export class JsonCodegen extends AbstractBinaryCodegen<JsonEncoder> {
142144
const ri = codegen.r();
143145
const rLength = codegen.r();
144146
const keys = fields.map((field) => JSON.stringify(field.key));
145-
const rKnownFields = codegen.addConstant(`new Set([${keys.join(',')}])`);
146-
codegen.js(`var ${rKeys} = Object.keys(${r}), ${rLength} = ${rKeys}.length, ${rKey};`);
147-
codegen.js(`for (var ${ri} = 0; ${ri} < ${rLength}; ${ri}++) {`);
148-
codegen.js(`${rKey} = ${rKeys}[${ri}];`);
149-
codegen.js(`if (${rKnownFields}.has(${rKey})) continue;`);
150-
codegen.js(`encoder.writeStr(${rKey});`);
147+
const rKnownFields = codegen.addConstant(/* js */ `new Set([${keys.join(',')}])`);
148+
codegen.js(/* js */ `var ${rKeys} = Object.keys(${r}), ${rLength} = ${rKeys}.length, ${rKey};`);
149+
codegen.js(/* js */ `for (var ${ri} = 0; ${ri} < ${rLength}; ${ri}++) {`);
150+
codegen.js(/* js */ `${rKey} = ${rKeys}[${ri}];`);
151+
codegen.js(/* js */ `if (${rKnownFields}.has(${rKey})) continue;`);
152+
codegen.js(/* js */ `encoder.writeStr(${rKey});`);
151153
this.blob(keySeparatorBlob);
152-
codegen.js(`encoder.writeAny(${r}[${rKey}]);`);
154+
codegen.js(/* js */ `encoder.writeAny(${r}[${rKey}]);`);
153155
this.blob(separatorBlob);
154-
codegen.js(`}`);
156+
codegen.js(/* js */ `}`);
155157
};
156158
const emitEnding = () => {
157159
const rewriteLastSeparator = () => {
158-
for (let i = 0; i < endBlob.length; i++) codegen.js(`uint8[writer.x - ${endBlob.length - i}] = ${endBlob[i]};`);
160+
for (let i = 0; i < endBlob.length; i++) codegen.js(/* js */ `uint8[writer.x - ${endBlob.length - i}] = ${endBlob[i]};`);
159161
};
160162
if (requiredFields.length) {
161163
rewriteLastSeparator();
162164
} else {
163165
codegen.if(
164-
`uint8[writer.x - 1] === ${separatorBlob[separatorBlob.length - 1]}`,
166+
/* js */ `uint8[writer.x - 1] === ${separatorBlob[separatorBlob.length - 1]}`,
165167
() => {
166168
rewriteLastSeparator();
167169
},
@@ -192,30 +194,29 @@ export class JsonCodegen extends AbstractBinaryCodegen<JsonEncoder> {
192194
}
193195

194196
protected onMap(path: SchemaPath, val: JsExpression, type: MapType): void {
195-
const objStartBlob = this.gen((encoder) => encoder.writeStartObj());
196-
const objEndBlob = this.gen((encoder) => encoder.writeEndObj());
197197
const separatorBlob = this.gen((encoder) => encoder.writeObjSeparator());
198198
const keySeparatorBlob = this.gen((encoder) => encoder.writeObjKeySeparator());
199199
const codegen = this.codegen;
200200
const r = codegen.var(val.use());
201201
const rKeys = codegen.var(`Object.keys(${r})`);
202202
const rKey = codegen.var();
203+
const ri = codegen.var();
203204
const rLength = codegen.var(`${rKeys}.length`);
204-
this.blob(objStartBlob);
205+
this.blob(this.gen((encoder) => encoder.writeStartObj()));
205206
codegen.if(`${rLength}`, () => {
206207
codegen.js(`${rKey} = ${rKeys}[0];`);
207208
codegen.js(`encoder.writeStr(${rKey});`);
208209
this.blob(keySeparatorBlob);
209210
this.onNode([...path, {r: rKey}], new JsExpression(() => `${r}[${rKey}]`), type._value);
210211
});
211-
codegen.js(`for (var i = 1; i < ${rLength}; i++) {`);
212-
codegen.js(`${rKey} = ${rKeys}[i];`);
212+
codegen.js(`for (var ${ri} = 1; ${ri} < ${rLength}; ${ri}++) {`);
213+
codegen.js(`${rKey} = ${rKeys}[${ri}];`);
213214
this.blob(separatorBlob);
214215
codegen.js(`encoder.writeStr(${rKey});`);
215216
this.blob(keySeparatorBlob);
216217
this.onNode([...path, {r: rKey}], new JsExpression(() => `${r}[${rKey}]`), type._value);
217218
codegen.js(`}`);
218-
this.blob(objEndBlob);
219+
this.blob(this.gen((encoder) => encoder.writeEndObj()));
219220
}
220221

221222
protected onKey(path: SchemaPath, r: JsExpression, type: KeyType<any, any>): void {

‎src/codegen/binary/json/__tests__/JsonCodegen.spec.ts‎

Lines changed: 84 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import {Writer} from '@jsonjoy.com/buffers/lib/Writer';
22
import {parse} from '@jsonjoy.com/json-pack/lib/json-binary';
33
import {JsonEncoder} from '@jsonjoy.com/json-pack/lib/json/JsonEncoder';
4-
import type {Type} from '../../../../type';
5-
import type {ModuleType} from '../../../../type/classes/ModuleType';
4+
import {ModuleType} from '../../../../type/classes/ModuleType';
65
import {testBinaryCodegen} from '../../__tests__/testBinaryCodegen';
76
import {JsonCodegen} from '../JsonCodegen';
7+
import type {Type} from '../../../../type';
88

99
const encoder = new JsonEncoder(new Writer(16));
1010

@@ -14,9 +14,91 @@ const transcode = (system: ModuleType, type: Type, value: unknown) => {
1414
fn(value, encoder);
1515
const encoded = encoder.writer.flush();
1616
const json = Buffer.from(encoded).toString('utf-8');
17+
// console.log(value);
1718
// console.log(json);
1819
const decoded = parse(json);
1920
return decoded;
2021
};
2122

2223
testBinaryCodegen(transcode);
24+
25+
test('fuzzer 1: map in map', () => {
26+
const system = new ModuleType();
27+
const {t} = system;
28+
const type = t.Map(t.Map(t.nil));
29+
const value = {
30+
'^': {
31+
'ww9DP[c': null,
32+
'2LL*vp ': null,
33+
'OW;a(w)': null,
34+
'T`jb_LZ': null,
35+
'C)crlQL': null,
36+
'kw&p(^-': null,
37+
'oKkF,u8': null
38+
}
39+
};
40+
const value2 = {
41+
'YS9mc}Zb': {
42+
'V2*_9': null,
43+
'j9?_0': null,
44+
'@:ODe': null,
45+
'sS{Sx': null,
46+
'4EMz|': null
47+
},
48+
"bF@64u'7": {
49+
'q<_b%}$Q': null,
50+
'RäXpXBLü': null,
51+
'$uJx]{ft': null,
52+
'bX%jLhr{': null,
53+
'Lr1bY-fY': null,
54+
'D]ml,C)W': null,
55+
'eK=DszFO': null,
56+
'!RqC^GUz': null
57+
},
58+
'9SEDa*#|': {
59+
';COK{m%=': null,
60+
'i`tJj:xE': null,
61+
'ffIhp!Om': null,
62+
'kiN&BfB5': null,
63+
'k+$es!mO': null,
64+
'O1(&D_bt': null,
65+
'cidA#*BD': null,
66+
'!ZP5JBFq': null
67+
},
68+
';6(7#5m:': {},
69+
'zhGX^&Y3': {
70+
'1Z>iC': null,
71+
'%вqL=': null,
72+
'5?5{)': null,
73+
'*2"H4': null,
74+
')&_O4': null
75+
},
76+
'?6a1a5Y\\': {
77+
'5,bCV': null,
78+
'z[x2s': null,
79+
'Ad/g9': null,
80+
'at#84': null,
81+
'{@?".': null
82+
},
83+
'uaaAwаHb': { VXy: null, 'I(<': null, 'W V': null },
84+
'&sH?Bk2E': {
85+
'M[^ex': null,
86+
'-ZP$E': null,
87+
'c*@uR': null,
88+
'`sy3N': null,
89+
'g?DB ': null
90+
}
91+
};
92+
const value3 = {
93+
'/7': { '|;L': null, '@K<': null, '*x:': null },
94+
Zf: { N1: null, 't%': null }
95+
};
96+
for (let i = 0; i < 100; i++) {
97+
const decoded = transcode(system, type, value);
98+
const decoded2 = transcode(system, type, value2);
99+
const decoded3 = transcode(system, type, value3);
100+
expect(decoded).toEqual(value);
101+
expect(decoded2).toEqual(value2);
102+
expect(decoded3).toEqual(value3);
103+
}
104+
});

‎src/codegen/binary/json/__tests__/automated.spec.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import {Writer} from '@jsonjoy.com/buffers/lib/Writer';
2+
import {JsonDecoder} from '@jsonjoy.com/json-pack/lib/json/JsonDecoder';
3+
import {JsonEncoder} from '@jsonjoy.com/json-pack/lib/json/JsonEncoder';
4+
import {parse} from '@jsonjoy.com/json-pack/lib/json-binary';
5+
import {JsonCodegen} from '../JsonCodegen';
6+
import {Random} from '../../../../random';
7+
import {allSerializableTypes} from '../../../../__tests__/fixtures';
8+
9+
const encoder = new JsonEncoder(new Writer(16));
10+
const decoder = new JsonDecoder();
11+
12+
for (const [name, type] of Object.entries(allSerializableTypes)) {
13+
test(`can encode and decode ${name}`, () => {
14+
for (let i = 0; i < 100; i++) {
15+
const json = Random.gen(type);
16+
// console.log(json);
17+
try {
18+
const fn = JsonCodegen.get(type);
19+
fn(json, encoder);
20+
const encoded = encoder.writer.flush();
21+
const text = Buffer.from(encoded).toString('utf-8');
22+
// console.log(text);
23+
const decoded = parse(text);
24+
// const decoded = decoder.decode(encoded);
25+
expect(decoded).toEqual(json);
26+
} catch (error) {
27+
console.log(JSON.stringify(json, null, 2));
28+
console.log(type + '');
29+
throw error;
30+
}
31+
}
32+
});
33+
}

‎src/codegen/binary/msgpack/__tests__/automated.spec.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import {MsgPackDecoder} from '@jsonjoy.com/json-pack/lib/msgpack/MsgPackDecoder';
2+
import {MsgPackEncoder} from '@jsonjoy.com/json-pack/lib/msgpack/MsgPackEncoder';
3+
import {MsgPackCodegen} from '../MsgPackCodegen';
4+
import {Random} from '../../../../random';
5+
import {allSerializableTypes} from '../../../../__tests__/fixtures';
6+
7+
const encoder = new MsgPackEncoder();
8+
const decoder = new MsgPackDecoder();
9+
10+
for (const [name, type] of Object.entries(allSerializableTypes)) {
11+
test(`can encode and decode ${name}`, () => {
12+
for (let i = 0; i < 100; i++) {
13+
const json = Random.gen(type);
14+
try {
15+
const fn = MsgPackCodegen.get(type);
16+
fn(json, encoder);
17+
const encoded = encoder.writer.flush();
18+
const decoded = decoder.decode(encoded);
19+
expect(decoded).toEqual(json);
20+
} catch (error) {
21+
console.log(JSON.stringify(json, null, 2));
22+
console.log(type + '');
23+
throw error;
24+
}
25+
}
26+
});
27+
}

‎src/codegen/capacity/__tests__/CapacityEstimatorCodegenContext.spec.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import {maxEncodingCapacity} from '@jsonjoy.com/util/lib/json-size';
22
import {t} from '../../../type';
33
import {ModuleType} from '../../../type/classes/ModuleType';
44
import {CapacityEstimatorCodegen} from '../CapacityEstimatorCodegen';
5+
import {Random} from '../../../random';
56

67
describe('"any" type', () => {
78
test('returns the same result as maxEncodingCapacity()', () => {
@@ -354,3 +355,14 @@ test('add circular reference test', () => {
354355
const estimator = CapacityEstimatorCodegen.get(user.type);
355356
expect(estimator(value1)).toBe(maxEncodingCapacity(value1));
356357
});
358+
359+
test('fuzzer: map in map', () => {
360+
const system = new ModuleType();
361+
const {t} = system;
362+
const type = t.Map(t.Map(t.nil));
363+
const estimator = CapacityEstimatorCodegen.get(type);
364+
for (let i = 0; i < 100; i++) {
365+
const value = Random.gen(type);
366+
expect(estimator(value)).toBe(maxEncodingCapacity(value));
367+
}
368+
});

‎src/type/classes/OrType.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ export class OrType<T extends Type[] = any> extends AbsType<schema.OrSchema<{[K
3232
Object.assign(this.schema, options);
3333
const discriminator = options.discriminator;
3434
if (discriminator) {
35-
if (discriminator.length === 2 && discriminator[0] === 'num' && discriminator[1] === -1) {
35+
if ((<any>discriminator === -1) || (discriminator.length === 2 && discriminator[0] === 'num' && discriminator[1] === -1)) {
3636
this.schema.discriminator = Discriminator.createExpression(this.types);
3737
}
3838
}

0 commit comments

Comments
 (0)