From 63a3c7e72526a2b7da99908680fccfd62df76070 Mon Sep 17 00:00:00 2001 From: David Worms Date: Thu, 14 Aug 2025 23:43:24 +0200 Subject: [PATCH] feat(csv-parse): with option `columns` (fix #461 #464 #466) --- packages/csv-parse/lib/index.d.ts | 21 +-- packages/csv-parse/lib/stream.d.ts | 1 + packages/csv-parse/lib/sync.d.ts | 11 +- packages/csv-parse/test/api.arguments.ts | 18 ++- packages/csv-parse/test/api.sync.ts | 160 +++++++++++++------- packages/csv-parse/test/api.types.ts | 22 ++- packages/csv-parse/test/option.on_record.ts | 29 ++++ 7 files changed, 184 insertions(+), 78 deletions(-) diff --git a/packages/csv-parse/lib/index.d.ts b/packages/csv-parse/lib/index.d.ts index 898f74b6..066bbb7f 100644 --- a/packages/csv-parse/lib/index.d.ts +++ b/packages/csv-parse/lib/index.d.ts @@ -93,7 +93,7 @@ export type ColumnOption = | false | { name: K }; -export interface OptionsNormalized { +export interface OptionsNormalized { auto_parse?: boolean | CastingFunction; auto_parse_date?: boolean | CastingDateFunction; /** @@ -190,7 +190,8 @@ export interface OptionsNormalized { /** * Alter and filter records by executing a user defined function. */ - on_record?: (record: T, context: InfoRecord) => T | undefined; + on_record?: (record: U, context: InfoRecord) => T | null | undefined; + // on_record?: (record: T, context: InfoRecord) => T | undefined; /** * Optional character surrounding a field, one character only, defaults to double quotes. */ @@ -258,7 +259,7 @@ Note, could not `extends stream.TransformOptions` because encoding can be BufferEncoding and undefined as well as null which is not defined in the extended type. */ -export interface Options { +export interface Options { /** * If true, the parser will attempt to convert read data types to native types. * @deprecated Use {@link cast} @@ -366,8 +367,8 @@ export interface Options { /** * Alter and filter records by executing a user defined function. */ - on_record?: (record: T, context: InfoRecord) => T | null | undefined; - onRecord?: (record: T, context: InfoRecord) => T | null | undefined; + on_record?: (record: U, context: InfoRecord) => T | null | undefined | U; + onRecord?: (record: U, context: InfoRecord) => T | null | undefined | U; /** * Function called when an error occured if the `skip_records_with_error` * option is activated. @@ -479,13 +480,13 @@ export class CsvError extends Error { ); } -type OptionsWithColumns = Omit, "columns"> & { +export type OptionsWithColumns = Omit, "columns"> & { columns: Exclude; }; -declare function parse( +declare function parse( input: string | Buffer | Uint8Array, - options: OptionsWithColumns, + options: OptionsWithColumns, callback?: Callback, ): Parser; declare function parse( @@ -494,8 +495,8 @@ declare function parse( callback?: Callback, ): Parser; -declare function parse( - options: OptionsWithColumns, +declare function parse( + options: OptionsWithColumns, callback?: Callback, ): Parser; declare function parse(options: Options, callback?: Callback): Parser; diff --git a/packages/csv-parse/lib/stream.d.ts b/packages/csv-parse/lib/stream.d.ts index b65263b8..fce4a613 100644 --- a/packages/csv-parse/lib/stream.d.ts +++ b/packages/csv-parse/lib/stream.d.ts @@ -11,6 +11,7 @@ export { ColumnOption, Options, OptionsNormalized, + OptionsWithColumns, Info, InfoCallback, InfoDataSet, diff --git a/packages/csv-parse/lib/sync.d.ts b/packages/csv-parse/lib/sync.d.ts index 4686a61b..be3dd727 100644 --- a/packages/csv-parse/lib/sync.d.ts +++ b/packages/csv-parse/lib/sync.d.ts @@ -1,12 +1,8 @@ -import { Options } from "./index.js"; +import { Options, OptionsWithColumns } from "./index.js"; -type OptionsWithColumns = Omit, "columns"> & { - columns: Exclude; -}; - -declare function parse( +declare function parse( input: Buffer | string | Uint8Array, - options: OptionsWithColumns, + options: OptionsWithColumns, ): T[]; declare function parse( input: Buffer | string | Uint8Array, @@ -24,6 +20,7 @@ export { ColumnOption, Options, OptionsNormalized, + OptionsWithColumns, Info, InfoCallback, InfoDataSet, diff --git a/packages/csv-parse/test/api.arguments.ts b/packages/csv-parse/test/api.arguments.ts index 93c91527..e3fc549b 100644 --- a/packages/csv-parse/test/api.arguments.ts +++ b/packages/csv-parse/test/api.arguments.ts @@ -89,7 +89,11 @@ describe("API arguments", function () { }); it("options:object, callback:function; write data and get result in callback", function (next) { - const parser = parse({ columns: true }, (err, records) => { + interface RowType { + field_1: string; + field_2: string; + } + const parser = parse({ columns: true }, (err, records) => { if (err) return next(err); records.should.eql([{ field_1: "value 1", field_2: "value 2" }]); next(); @@ -123,7 +127,11 @@ describe("API arguments", function () { describe("3 args", function () { it("data:string, options:object, callback:function", function (next) { - parse( + interface RowType { + field_1: string; + field_2: string; + } + parse( "field_1,field_2\nvalue 1,value 2", { columns: true }, (err, records) => { @@ -135,7 +143,11 @@ describe("API arguments", function () { }); it("data:buffer, options:object, callback:function", function (next) { - parse( + interface RowType { + field_1: string; + field_2: string; + } + parse( Buffer.from("field_1,field_2\nvalue 1,value 2"), { columns: true }, (err, records) => { diff --git a/packages/csv-parse/test/api.sync.ts b/packages/csv-parse/test/api.sync.ts index d04a3f6e..4f439db8 100644 --- a/packages/csv-parse/test/api.sync.ts +++ b/packages/csv-parse/test/api.sync.ts @@ -3,74 +3,120 @@ import dedent from "dedent"; import { parse } from "../lib/sync.js"; describe("API sync", function () { - it("take a string and return records", function () { - const records = parse("field_1,field_2\nvalue 1,value 2"); - records.should.eql([ - ["field_1", "field_2"], - ["value 1", "value 2"], - ]); - }); + describe("content", function () { + it("take a string and return records", function () { + const records = parse("field_1,field_2\nvalue 1,value 2"); + records.should.eql([ + ["field_1", "field_2"], + ["value 1", "value 2"], + ]); + }); - it("take a buffer and return records", function () { - const records = parse(Buffer.from("field_1,field_2\nvalue 1,value 2")); - records.should.eql([ - ["field_1", "field_2"], - ["value 1", "value 2"], - ]); - }); + it("take a buffer and return records", function () { + const records = parse(Buffer.from("field_1,field_2\nvalue 1,value 2")); + records.should.eql([ + ["field_1", "field_2"], + ["value 1", "value 2"], + ]); + }); - it("take a Uint8Array and return records", function () { - const records = parse( - new TextEncoder().encode("field_1,field_2\nvalue 1,value 2"), - ); - records.should.eql([ - ["field_1", "field_2"], - ["value 1", "value 2"], - ]); + it("take a Uint8Array and return records", function () { + const records = parse( + new TextEncoder().encode("field_1,field_2\nvalue 1,value 2"), + ); + records.should.eql([ + ["field_1", "field_2"], + ["value 1", "value 2"], + ]); + }); }); - it("honors columns option", function () { - const records = parse("field_1,field_2\nvalue 1,value 2", { - columns: true, + describe("options", function () { + it("`columns` option without generic", function () { + // Parse returns unknown[] + const records = parse("field_1,field_2\nvalue 1,value 2", { + columns: true, + }); + records.should.eql([{ field_1: "value 1", field_2: "value 2" }]); + }); + + it("`columns` option with generic", function () { + // Parse returns Record[] + interface Record { + field_1: string; + field_2: string; + } + const records: Record[] = parse( + "field_1,field_2\nvalue 1,value 2", + { + columns: true, + }, + ); + records.should.eql([{ field_1: "value 1", field_2: "value 2" }]); }); - records.should.eql([{ field_1: "value 1", field_2: "value 2" }]); - }); - it("honors objname option", function () { - const records = parse("field_1,field_2\nname 1,value 1\nname 2,value 2", { - objname: "field_1", - columns: true, + it("`columns` and `on_record` options with generic", function () { + // Parse returns Record[] + interface RecordOriginal { + field_a: string; + field_b: string; + } + interface Record { + field_1: string; + field_2: string; + } + const records: Record[] = parse( + "field_a,field_b\nvalue 1,value 2", + { + columns: true, + on_record: (record: RecordOriginal) => ({ + field_1: record.field_a, + field_2: record.field_b, + }), + }, + ); + records.should.eql([{ field_1: "value 1", field_2: "value 2" }]); }); - records.should.eql({ - "name 1": { field_1: "name 1", field_2: "value 1" }, - "name 2": { field_1: "name 2", field_2: "value 2" }, + + it("`objname` option", function () { + // Not good, parse returns unknown[] + const records = parse("field_1,field_2\nname 1,value 1\nname 2,value 2", { + objname: "field_1", + columns: true, + }); + records.should.eql({ + "name 1": { field_1: "name 1", field_2: "value 1" }, + "name 2": { field_1: "name 2", field_2: "value 2" }, + }); }); - }); - it("honors to_line", function () { - const records = parse("1\n2\n3\n4", { to_line: 2 }); - records.should.eql([["1"], ["2"]]); + it("`to_line` option", function () { + const records = parse("1\n2\n3\n4", { to_line: 2 }); + records.should.eql([["1"], ["2"]]); + }); }); - it("catch errors", function () { - try { - parse("A,B\nB\nC,K", { trim: true }); - throw Error("Error not catched"); - } catch (err) { - if (!err) throw Error("Invalid assessment"); - (err as Error).message.should.eql( - "Invalid Record Length: expect 2, got 1 on line 2", - ); - } - }); + describe("errors", function () { + it("catch errors", function () { + try { + parse("A,B\nB\nC,K", { trim: true }); + throw Error("Error not catched"); + } catch (err) { + if (!err) throw Error("Invalid assessment"); + (err as Error).message.should.eql( + "Invalid Record Length: expect 2, got 1 on line 2", + ); + } + }); - it("catch err in last line while flushing", function () { - (() => { - parse(dedent` - headerA, headerB - A2, B2 - A1, B1, C2, D2 - `); - }).should.throw("Invalid Record Length: expect 2, got 4 on line 3"); + it("catch err in last line while flushing", function () { + (() => { + parse(dedent` + headerA, headerB + A2, B2 + A1, B1, C2, D2 + `); + }).should.throw("Invalid Record Length: expect 2, got 4 on line 3"); + }); }); }); diff --git a/packages/csv-parse/test/api.types.ts b/packages/csv-parse/test/api.types.ts index 12821ae9..f924c96a 100644 --- a/packages/csv-parse/test/api.types.ts +++ b/packages/csv-parse/test/api.types.ts @@ -487,7 +487,7 @@ describe("API Types", function () { { columns: true, }, - (error, records: unknown[] | undefined) => { + (error, records: unknown[]) => { records; next(error); }, @@ -506,5 +506,25 @@ describe("API Types", function () { }, ); }); + + it("Exposes U[] and T if columns and on_record are specified", function (next) { + type PersonOriginal = { surname: string; age: number }; + parse( + "", + { + columns: true, + on_record: (record: PersonOriginal) => { + return { + name: record.surname, + age: record.age, + }; + }, + }, + (error, records: Person[]) => { + records; + next(error); + }, + ); + }); }); }); diff --git a/packages/csv-parse/test/option.on_record.ts b/packages/csv-parse/test/option.on_record.ts index a282a1bb..af68d458 100644 --- a/packages/csv-parse/test/option.on_record.ts +++ b/packages/csv-parse/test/option.on_record.ts @@ -8,6 +8,7 @@ describe("Option `on_record`", function () { "a,b", { on_record: (record) => { + console.log(record[1]); return [record[1], record[0]]; }, }, @@ -86,6 +87,34 @@ describe("Option `on_record`", function () { }, ); }); + + it("with option `columns` (fix #461 #464 #466)", function (next) { + type RowTypeOriginal = { a: string; b: string }; + type RowTypeFinal = { prop_a: string; prop_b: string }; + parse( + "a,1\nb,2", + { + on_record: (record: RowTypeOriginal): RowTypeFinal => ({ + prop_a: record.a, + prop_b: record.b, + }), + columns: ["a", "b"], + }, + function (err, records: RowTypeFinal[]) { + records.should.eql([ + { + prop_a: "a", + prop_b: "1", + }, + { + prop_a: "b", + prop_b: "2", + }, + ]); + next(); + }, + ); + }); }); describe("context", function () {