Merge pull request #23 from mrodrig/csv2json-wrap-fix

csv2json Wrapped Field Support

Author: mrodrig

README.md

@@ -90,11 +90,12 @@ BMW,X5,2014,3287,M
   * `DELIMITER` - Document - Specifies the different types of delimiters
     * `FIELD` - String - Field Delimiter. Default: `','`
     * `ARRAY` - String - Array Value Delimiter. Default: `';'`
+    * `WRAP` - String - The character that field values are wrapped in. Default: `''`
   * `EOL` - String - End of Line Delimiter. Default: `'\n'`
   * `PARSE_CSV_NUMBERS` - Boolean - (TODO) Should numbers that are found in the CSV be converted to numbers? Default: `false`
   * `KEYS` - Array - Specify the keys (as strings) that should be converted. Default: `null`
     * If you have a nested object (ie. {info : {name: 'Mike'}}), then set options.KEYS to ['info.name']
-    * If you want all keys to be converted, then specify ```null``` or don't specify the option to utilize the default.
+    * If you want all keys to be converted, then specify `null` or don't specify the option to utilize the default.
 
 ##### csv2json Example:
 
@@ -145,10 +146,10 @@ $ npm run coverage
 
 Current Coverage is:
 ```
-Statements   : 94.49% ( 120/127 )
-Branches     : 90.24% ( 74/82 )
-Functions    : 100% ( 30/30 )
-Lines        : 97.39% ( 112/115 )
+Statements   : 96.88% ( 155/160 )
+Branches     : 93.55% ( 116/124 )
+Functions    : 100% ( 31/31 )
+Lines        : 97.99% ( 146/149 )
 ```
 
 ## Features
@@ -162,6 +163,7 @@ Lines        : 97.39% ( 112/115 )
 - Ability to re-generate the JSON documents that were used to generate the CSV (including nested documents)
 - Allows for custom field delimiters, end of line delimiters, etc.
 - Promisifiable via bluebird's .promisify(<function>) and .promisifyAll(<object>) (as of 1.1.1)
+- Wrapped value support for json2csv and csv2json (as of 1.3.0)
 
 ## F.A.Q.
 

bower.json

@@ -1,6 +1,6 @@
 {
   "name": "json-2-csv",
-  "version": "1.2.1",
+  "version": "1.3.0",
   "homepage": "https://github.com/mrodrig/json-2-csv",
   "moduleType": [
     "node"

lib/csv-2-json.js

@@ -19,7 +19,7 @@ var retrieveHeading = function (lines, callback) {
     }
 
     // Generate and return the heading keys
-    return _.map(lines[0].split(options.DELIMITER.FIELD),
+    return _.map(splitLine(lines[0]),
         function (headerKey, index) {
             return {
                 value: headerKey,
@@ -64,7 +64,7 @@ var convertArrayRepresentation = function (arrayRepresentation) {
  * @returns {Object} created json document
  */
 var createDocument = function (keys, line) {
-    var line = line.split(options.DELIMITER.FIELD), // Split the line using the given field delimiter after trimming whitespace
+    var line = splitLine(line), // Split the line using the given field delimiter after trimming whitespace
         val; // Temporary variable to set the current key's value to
 
     // Reduce the keys into a JSON document representing the given line
@@ -102,6 +102,83 @@ var convertCSV = function (lines, callback) {
     }, []);
 };
 
+/**
+ * Helper function that splits a line so that we can handle wrapped fields
+ * @param line
+ */
+var splitLine = function (line) {
+    // If the fields are not wrapped, return the line split by the field delimiter
+    if (!options.DELIMITER.WRAP) { return line.split(options.DELIMITER.FIELD); }
+
+    // Parse out the line...
+    var splitLine = [],
+        character,
+        charBefore,
+        charAfter,
+        lastCharacterIndex = line.length - 1,
+        stateVariables = {
+            insideWrapDelimiter: false,
+            parsingValue: true,
+            startIndex: 0
+        },
+        index = 0;
+
+    // Loop through each character in the line to identify where to split the values
+    while(index < line.length) {
+        // Current character
+        character  = line[index];
+        // Previous character
+        charBefore = index ? line[index - 1] : '';
+        // Next character
+        charAfter = index < lastCharacterIndex ? line[index + 1] : '';
+
+        // If we reached the end of the line, add the remaining value
+        if (index === lastCharacterIndex) {
+            splitLine.push(line.substring(stateVariables.startIndex, stateVariables.insideWrapDelimiter ? index : undefined));
+        }
+        // If the line starts with a wrap delimiter
+        else if (character === options.DELIMITER.WRAP && index === 0) {
+            stateVariables.insideWrapDelimiter = true;
+            stateVariables.parsingValue = true;
+            stateVariables.startIndex = index + 1;
+        }
+
+        // If we reached a wrap delimiter with a field delimiter after it (ie. *",)
+        else if (character === options.DELIMITER.WRAP && charAfter === options.DELIMITER.FIELD) {
+            splitLine.push(line.substring(stateVariables.startIndex, index));
+            stateVariables.startIndex = index + 2; // next value starts after the field delimiter
+            stateVariables.insideWrapDelimiter = false;
+            stateVariables.parsingValue = false;
+        }
+        // If we reached a wrap delimiter with a field delimiter after it (ie. ,"*)
+        else if (character === options.DELIMITER.WRAP && charBefore === options.DELIMITER.FIELD) {
+            if (stateVariables.parsingValue) {
+                splitLine.push(line.substring(stateVariables.startIndex, index-1));
+            }
+            stateVariables.insideWrapDelimiter = true;
+            stateVariables.parsingValue = true;
+            stateVariables.startIndex = index + 1;
+        }
+        // If we reached a field delimiter and are not inside the wrap delimiters (ie. *,*)
+        else if (character === options.DELIMITER.FIELD && charBefore !== options.DELIMITER.WRAP
+                    && charAfter !== options.DELIMITER.WRAP && !stateVariables.insideWrapDelimiter
+                    && stateVariables.parsingValue) {
+            splitLine.push(line.substring(stateVariables.startIndex, index));
+            stateVariables.startIndex = index + 1;
+        }
+        else if (character === options.DELIMITER.FIELD && charBefore === options.DELIMITER.WRAP
+                    && charAfter !== options.DELIMITER.WRAP) {
+            stateVariables.insideWrapDelimiter = false;
+            stateVariables.parsingValue = true;
+            stateVariables.startIndex = index + 1;
+        }
+        // Otherwise increment to the next character
+        index++;
+    }
+
+    return splitLine;
+};
+
 module.exports = {
 
     /**
@@ -116,11 +193,11 @@ module.exports = {
         if (!callback) { throw new Error(constants.Errors.callbackRequired); }
 
         // Shouldn't happen, but just in case
-        if (!opts) { return callback(new Error(constants.Errors.optionsRequired)); return null; }
+        if (!opts) { return callback(new Error(constants.Errors.optionsRequired)); }
         options = opts; // Options were passed, set the global options value
 
         // If we don't receive data, report an error
-        if (!data) { return callback(new Error(constants.Errors.csv2json.cannotCallCsv2JsonOn + data + '.')); return null; }
+        if (!data) { return callback(new Error(constants.Errors.csv2json.cannotCallCsv2JsonOn + data + '.')); }
 
         // The data provided is not a string
         if (!_.isString(data)) {

package.json

@@ -2,7 +2,7 @@
     "author": "mrodrig",
     "name": "json-2-csv",
     "description": "A JSON to CSV and CSV to JSON converter that natively supports sub-documents and auto-generates the CSV heading.",
-    "version": "1.2.1",
+    "version": "1.3.0",
     "repository": {
         "type": "git",
     	"url": "http://github.com/mrodrig/json-2-csv.git"

test.js

@@ -0,0 +1,6 @@
+"carModel","priceRange.min","priceRange.max"
+Audi,"9000",11000
+"BMW",14000,16000
+"Mercedes",19000,"21000"
+"Porsche","29000",31000
+,200,300

test/CSV/quoted/nestedSomeFieldsWrappedJson.csv

@@ -0,0 +1,37 @@
+[
+  {
+	"carModel": "Audi",
+	"priceRange": {
+	  "min": "9000",
+	  "max": "11000"
+	}
+  },
+  {
+	"carModel": "BMW",
+	"priceRange": {
+	  "min": "14000",
+	  "max": "16000"
+	}
+  },
+  {
+	"carModel": "Mercedes",
+	"priceRange": {
+	  "min": "19000",
+	  "max": "21000"
+	}
+  },
+  {
+	"carModel": "Porsche",
+	"priceRange": {
+	  "min": "29000",
+	  "max": "31000"
+	}
+  },
+  {
+	"carModel": null,
+	"priceRange": {
+	  "min": "200",
+	  "max": "300"
+	}
+  }
+]