Headerless CSV support (#229)

mrodrig · web-flow · commit d8804ae0dbf7 · 2023-02-19T22:51:27.000-05:00
* feat: Add option to support headerless CSV fixes #227 * test/doc: add tests and docs for headerFields option * docs: document headerFields option * ci: remove unused coveralls file * ci: add github actions to run tests * chore(rel): 3.19.0
diff --git a/.coveralls.yml b/.coveralls.yml
diff --git a/.github/workflows/automated-tests-workflow.yml b/.github/workflows/automated-tests-workflow.yml
@@ -0,0 +1,42 @@
+name: Automated Node Version Tests
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node: [16, 18, 19]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node ${{ matrix.node }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+      - name: Install NPM Modules
+        run: npm ci
+      - name: Lint
+        run: npm run lint
+      - name: Test
+        run: |
+          npm run coverage
+      - name: Coveralls Parallel
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.github_token }}
+          flag-name: node-${{ runner.os }}-${{ matrix.node }}
+          parallel: true
+  
+  coveralls-finished:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Coveralls Finished
+        uses: coverallsapp/github-action@master
+        with:
+          github-token: ${{ secrets.github_token }}
+          parallel-finished: true
diff --git a/README.md b/README.md
@@ -164,6 +164,10 @@ Available in version `2.2.0`, this functionality makes use of promises from the
       * Default: `\n`
   * `excelBOM` - Boolean - Does the CSV contain a unicode character prepended in order to allow Excel to open a UTF-8 encoded file with non-ASCII characters present?
     * Default: `false`
+  * `headerFields` - Array - Specify the field names (as strings) in place of a header line in the CSV itself.
+    * Default: Parses the header fields directly from the CSV string
+    * If you want to generate a nested object (ie. `{info : {name: 'Mike'}}`), then use `.` characters in the string to denote a nested field, like ['info.name']
+    * If your CSV has a header line included, then don't specify the option to utilize the default values that will be parsed from the CSV.
   * `keys` - Array - Specify the keys (as strings) that should be converted. 
     * Default: `null`
     * If you have a nested object (ie. `{info : {name: 'Mike'}}`), then set this to `['info.name']`
diff --git a/lib/converter.d.ts b/lib/converter.d.ts
@@ -1,4 +1,4 @@
-export interface ISharedOptions {
+interface SharedConverterOptions {
   /**
    * Specifies the different types of delimiters
    */
@@ -56,7 +56,17 @@ export interface ISharedOptions {
   preventCsvInjection?: boolean;
 }
 
-export interface IFullOptions extends ISharedOptions {
+export interface Csv2JsonOptions extends SharedConverterOptions {
+  /**
+   * Specify the header fields in the event that the CSV does not container a header line
+   *
+   * If you want to generate a nested object (ie. {info : {name: 'Mike'}}), then use `.` characters in the string to denote a nested field, like ['info.name']
+   * If your CSV has a header line included, then don't specify the option to utilize the default values that will be parsed from the CSV.
+   */
+  headerFields?: string[];
+}
+
+export interface Json2CsvOptions extends SharedConverterOptions {
   /**
    * Should all documents have the same schema?
    * @default false
@@ -121,11 +131,11 @@ export interface IFullOptions extends ISharedOptions {
 }
 
 export function json2csv(data: object[],
-                         callback: (err?: Error, csv?: string) => void, options?: IFullOptions): void;
+                         callback: (err?: Error, csv?: string) => void, options?: Json2CsvOptions): void;
 
-export function json2csvAsync(data: object[], options?: IFullOptions): Promise<string>;
+export function json2csvAsync(data: object[], options?: Json2CsvOptions): Promise<string>;
 
 export function csv2json(csv: string,
-                         callback: (err?: Error, data?: any[]) => void, options?: ISharedOptions): void;
+                         callback: (err?: Error, data?: any[]) => void, options?: Csv2JsonOptions): void;
 
-export function csv2jsonAsync(csv: string, options?: ISharedOptions): Promise<any[]>;
+export function csv2jsonAsync(csv: string, options?: Csv2JsonOptions): Promise<any[]>;
diff --git a/lib/csv2json.js b/lib/csv2json.js
@@ -30,17 +30,25 @@ const Csv2Json = function(options) {
      * @returns {*}
      */
     function retrieveHeading(lines) {
-        let params = {lines},
+        let params = {lines};
+
+        if (options.headerFields) {
+            params.headerFields = options.headerFields.map((headerField, index) => ({
+                value: processHeaderKey(headerField),
+                index
+            }));
+        } else {
             // Generate and return the heading keys
-            headerRow = params.lines[0];
-        params.headerFields = headerRow.map((headerKey, index) => ({
-            value: processHeaderKey(headerKey),
-            index: index
-        }));
-
-        // If the user provided keys, filter the generated keys to just the user provided keys so we also have the key index
-        if (options.keys) {
-            params.headerFields = params.headerFields.filter((headerKey) => options.keys.includes(headerKey.value));
+            let headerRow = params.lines[0];
+            params.headerFields = headerRow.map((headerKey, index) => ({
+                value: processHeaderKey(headerKey),
+                index
+            }));
+
+            // If the user provided keys, filter the generated keys to just the user provided keys so we also have the key index
+            if (options.keys) {
+                params.headerFields = params.headerFields.filter((headerKey) => options.keys.includes(headerKey.value));
+            }
         }
 
         return params;
@@ -227,7 +235,11 @@ const Csv2Json = function(options) {
      * @returns {*}
      */
     function retrieveRecordLines(params) {
-        params.recordLines = params.lines.splice(1); // All lines except for the header line
+        if (options.headerFields) { // This option is passed for instances where the CSV has no header line
+            params.recordLines = params.lines;
+        } else { // All lines except for the header line
+            params.recordLines = params.lines.splice(1);
+        }
 
         return params;
     }
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -5,7 +5,7 @@
     },
     "name": "json-2-csv",
     "description": "A JSON to CSV and CSV to JSON converter that natively supports sub-documents and auto-generates the CSV heading.",
-    "version": "3.18.1",
+    "version": "3.19.0",
     "homepage": "https://mrodrig.github.io/json-2-csv",
     "repository": {
         "type": "git",
diff --git a/test/config/testJsonFilesList.js b/test/config/testJsonFilesList.js
@@ -25,6 +25,7 @@ module.exports = {
     csvEmptyLastValue: require('../data/json/csvEmptyLastValue'),
     unwind: require('../data/json/unwind'),
     unwindEmptyArray: require('../data/json/unwindEmptyArray'),
+    unwindWithSpecifiedKeys: require('../data/json/unwindWithSpecifiedKeys'),
     localeFormat: require('../data/json/localeFormat'),
     invalidParsedValues: require('../data/json/invalidParsedValues'),
     firstColumnWrapCRLF: require('../data/json/firstColumnWrapCRLF.json'),
diff --git a/test/csv2json.js b/test/csv2json.js
@@ -449,6 +449,59 @@ function runTests(jsonTestData, csvTestData) {
                     parseValue: () => 'Parsed Value'
                 });
             });
+
+            it('should be able to parse CSV with no header line provided', (done) => {
+                const headerlessCsv = csvTestData.unwindWithSpecifiedKeys.split(options.delimiter.eol)
+                    .splice(1)
+                    .join(options.delimiter.eol);
+
+                converter.csv2json(headerlessCsv, (err, json) => {
+                    if (err) done(err);
+                    json.should.deepEqual(jsonTestData.unwindWithSpecifiedKeys);
+                    done();
+                }, {
+                    headerFields: ['data.category', 'data.options.name']
+                });
+            });
+
+            it('should be able to parse CSV with no header line provided and different header fields', (done) => {
+                const headerlessCsv = csvTestData.unwindWithSpecifiedKeys.split(options.delimiter.eol)
+                        .splice(1)
+                        .join(options.delimiter.eol),
+                    expectedJson = jsonTestData.unwindWithSpecifiedKeys.map((doc) => {
+                        doc.category = doc.data.category;
+                        doc.product = doc.data.options.name;
+                        delete doc.data;
+                        return doc;
+                    });
+
+                converter.csv2json(headerlessCsv, (err, json) => {
+                    if (err) done(err);
+                    json.should.deepEqual(expectedJson);
+                    done();
+                }, {
+                    headerFields: ['category', 'product']
+                });
+            });
+
+            it('should be able to parse CSV with no header line provided and only some of the fields', (done) => {
+                const headerlessCsv = csvTestData.unwindWithSpecifiedKeys.split(options.delimiter.eol)
+                        .splice(1)
+                        .join(options.delimiter.eol),
+                    expectedJson = jsonTestData.unwindWithSpecifiedKeys.map((doc) => {
+                        doc.category = doc.data.category;
+                        delete doc.data;
+                        return doc;
+                    });
+
+                converter.csv2json(headerlessCsv, (err, json) => {
+                    if (err) done(err);
+                    json.should.deepEqual(expectedJson);
+                    done();
+                }, {
+                    headerFields: ['category']
+                });
+            });
         });
     });
 
diff --git a/test/data/json/unwindWithSpecifiedKeys.json b/test/data/json/unwindWithSpecifiedKeys.json
@@ -0,0 +1,34 @@
+[
+    {
+        "data": {
+            "category": "Computers",
+            "options": {
+                "name": "MacBook Pro 15"
+            }
+        }
+    },
+    {
+        "data": {
+            "category": "Computers",
+            "options": {
+                "name": "MacBook Air 13"
+            }
+        }
+    },
+    {
+        "data": {
+            "category": "Cars",
+            "options": {
+                "name": "Supercharger"
+            }
+        }
+    },
+    {
+        "data": {
+            "category": "Cars",
+            "options": {
+                "name": "Turbocharger"
+            }
+        }
+    }
+]
