Handle nested EOL delimiter inside wrap delimiter properly (#215)

* chore(test): Add test case for #214

* Handle nested newlines in quoted fields.

As reported in #214, there was a bug with how the CSV parser was handling quoted fields which had newlines (or EOL delimiters) embedded in them. Since the value is wrapped in quotes, the EOL delimiter should not be interpreted as though it's a new line or the start of another field as it previously was. This commit introduces a fix to ensure that quoted fields are parsed properly regardless of whether there is a nested EOL delimiter inside.

Fixes #214

Author: mrodrig

lib/csv2json.js

@@ -162,7 +162,7 @@ const Csv2Json = function(options) {
                 stateVariables.insideWrapDelimiter = false;
                 stateVariables.parsingValue = false;
                 // Next iteration will substring, add the value to the line, and push the line onto the array of lines
-            } else if (character === options.delimiter.wrap && (index === 0 || utils.getNCharacters(csv, index - eolDelimiterLength, eolDelimiterLength) === options.delimiter.eol)) {
+            } else if (character === options.delimiter.wrap && (index === 0 || utils.getNCharacters(csv, index - eolDelimiterLength, eolDelimiterLength) === options.delimiter.eol && !stateVariables.insideWrapDelimiter)) {
                 // If the line starts with a wrap delimiter (ie. "*)
 
                 stateVariables.insideWrapDelimiter = true;
@@ -190,7 +190,7 @@ const Csv2Json = function(options) {
                 stateVariables.insideWrapDelimiter = true;
                 stateVariables.parsingValue = true;
                 stateVariables.startIndex = index;
-            } else if (character === options.delimiter.wrap && charAfter === options.delimiter.wrap) {
+            } else if (character === options.delimiter.wrap && charAfter === options.delimiter.wrap && index !== stateVariables.startIndex) {
                 // If we run into an escaped quote (ie. "") skip past the second quote
 
                 index += 2;

test/config/testCsvFilesList.js

@@ -44,7 +44,8 @@ const fs = require('fs'),
         {key: 'nestedDotKeys', file: '../data/csv/nestedDotKeys.csv'},
         {key: 'nestedDotKeysWithArray', file: '../data/csv/nestedDotKeysWithArray.csv'},
         {key: 'nestedDotKeysWithArrayExpandedUnwound', file: '../data/csv/nestedDotKeysWithArrayExpandedUnwound.csv'},
-        {key: 'emptyColumns', file: '../data/csv/emptyColumns.csv'}
+        {key: 'emptyColumns', file: '../data/csv/emptyColumns.csv'},
+        {key: 'quotedFieldWithNewline', file: '../data/csv/quotedFieldWithNewline.csv'}
     ];
 
 function readCsvFile(filePath) {

test/config/testJsonFilesList.js

@@ -34,5 +34,6 @@ module.exports = {
     nestedDotKeys: require('../data/json/nestedDotKeys.json'),
     nestedDotKeysWithArray: require('../data/json/nestedDotKeysWithArray.json'),
     nestedDotKeysWithArrayExpandedUnwound: require('../data/json/nestedDotKeysWithArrayExpandedUnwound.json'),
-    emptyColumns: require('../data/json/emptyColumns.json')
+    emptyColumns: require('../data/json/emptyColumns.json'),
+    quotedFieldWithNewline: require('../data/json/quotedFieldWithNewline.json')
 };

test/csv2json.js

@@ -226,13 +226,23 @@ function runTests(jsonTestData, csvTestData) {
                 });
             });
 
+            // Test case for #204
             it('should drop any values with empty column headers', (done) => {
                 converter.csv2json(csvTestData.emptyColumns, (err, json) => {
                     if (err) done(err);
                     json.should.deepEqual(jsonTestData.emptyColumns);
                     done();
                 });
             });
+
+            // Test case for #214
+            it('should handle quoted fields with nested EOL characters', (done) => {
+                converter.csv2json(csvTestData.quotedFieldWithNewline, (err, json) => {
+                    if (err) done(err);
+                    json.should.deepEqual(jsonTestData.quotedFieldWithNewline);
+                    done();
+                });
+            });
         });
 
         describe('Error Handling', () => {

test/data/csv/quotedFieldWithNewline.csv

@@ -0,0 +1,8 @@
+first,second,third
+"hello
+",world,test
+first,"world
+
+",test
+world,test,"hello
+"

test/data/json/quotedFieldWithNewline.json

@@ -0,0 +1,5 @@
+[ 
+    { "first": "hello\n", "second": "world", "third": "test" },
+    { "first": "first", "second": "world\n\n", "third": "test"},
+    { "first": "world", "second": "test", "third": "hello\n"}
+ ]