fixed a bug with line break; refactored line population

Author: Keyang

libs/core/csvConverter.js

@@ -34,7 +34,8 @@ function csvAdv(params) {
     this.resultObject.disableConstruct();
   }
   this.headRow = [];
-  this._buffer = "";
+  this._buffer = ""; //line buffer 
+  this._recordBuffer=""; //record buffer
   this.rowIndex = 0;
   this._isStarted = false;
   var self = this;
@@ -58,6 +59,21 @@ csvAdv.prototype._isToogleQuote = function(segment) {
     return false;
   }
 }
+//on line poped
+csvAdv.prototype._line=function(line,lastLine){
+  this._recordBuffer+=line;
+  if (!this._isToogleQuote(this._recordBuffer)){ //if a complete record is in buffer. start the parse
+   var data=this._recordBuffer;
+   this._recordBuffer="";
+   this._record(data,this.rowIndex++,lastLine) ;
+  }else{ //if the record in buffer is not a complete record (quote does not match). wait next line
+    this._recordBuffer+=this.eol;
+   if (lastLine){
+     throw ("Incomplete CSV file detected. Quotes does not match in pairs.");
+   }
+   return;
+  }
+}
 csvAdv.prototype._transform = function(data, encoding, cb) {
   var self = this;
   if (encoding == "buffer") {
@@ -79,23 +95,26 @@ csvAdv.prototype._transform = function(data, encoding, cb) {
   }
   if (this.eol) {
     //console.log(this._buffer);
-    if (this._buffer.indexOf(this.eol) > -1) {
+    if (this._buffer.indexOf(this.eol) > -1) { //if current data contains 1..* line break 
       var arr = this._buffer.split(this.eol);
       while (arr.length > 1) {
         var data = arr.shift();
-        if (data.length > 0) {
-          this.emit("record", data, this.rowIndex++);
-        }
+        this._line(data);
+          //this.emit("record", data, this.rowIndex++);
       }
-      this._buffer = arr[0];
+      this._buffer = arr[0]; //whats left (maybe half line). push to buffer
+      cb();
+    }else{ //if there is no line break appeared. wait next loop until it appears.
+      cb();
     }
 
+  }else{
+    cb();
   }
-  cb();
 };
 csvAdv.prototype._flush = function(cb) {
-  if (this._buffer.length != 0) { //emit last line
-    this.emit("record", this._buffer, this.rowIndex++, true);
+  if (this._buffer.length != 0) { //finished but still has buffer data. emit last line
+    this._line(this._buffer,  true);
   }
   cb();
 };

libs/core/init_onrecord.js

@@ -4,7 +4,7 @@
 var os = require("os");
 module.exports = function() {
   var self = this;
-  self.on("record", function(rowStr, index, lastLine) {
+  this._record=function(rowStr, index, lastLine) {
     var quote = self.param.quote;
     var delimiter = self.param.delimiter;
     var rowArr = rowStr.split(delimiter);
@@ -49,5 +49,5 @@ module.exports = function() {
       self.emit("record_parsed", resultRow, row, index - 1);
       self.push(JSON.stringify(resultRow),"utf8");
     }
-  });
+  };
 }

package.json

@@ -12,7 +12,7 @@
     "name": "Keyang Xiang",
     "email": "[email protected]"
   }],
-  "version": "0.3.16",
+  "version": "0.3.17",
   "keywords": [
     "csv",
     "json",

test/data/lineBreak

@@ -0,0 +1,8 @@
+F_Year,Event ID,Event Date,Time,Location,Rotue,TOC,Person type,Injury degree,FWI,Precursor,Gender,Under 16,Apparent age,Fatal,Alcohol/Drugs,Impairment,Encumbrances & Group Travel,Risk-taking Behaviour,Sub-standard conditions,Design,Operational error,Crowd management,Non-standard operation,Rushing,3rd Party Behaviour,Narrative Full,Period
+2013/2014,2914930,3/2/14,23:02:00,Pembrey and Burry Port,Wales,Arriva Trains Wales,Passenger,Minor non-reportable,0.001,Passenger falls from platform onto track (no electric shock nor struck by train) under the influence,Male,,Unknown,,x,,,,,,,,,,,"A report was initially received from Dyfed Powys Police that a person had been struck by a train at Pembrey and Burry Port station and that an ambulance was in attendance (the ambulance was the source of the police report). It later transpired that the person was inebriated and had fallen off of Pembrey platform after attempting to board the train after the doors were closed and the train was ready to depart. It has been confirmed (by Pembrey signaller) that the person has not been struck by the train and has NOT fallen under the train. One police officer on site. 
+
+Later advice from the train crew working 2B21 2135 Milford Haven to Cardiff Central stated the person had attempted to board 2B21 as it was ready to depart from the platform, bounced off of the train falling backwards, had then gotten back up and then fallen off of the platform and onto the track behind the train. The persons injuries (if any) have not been disclosed.
+
+The person was been removed from the track and normal working resumed from 2326. The on call MOM was called to attend and act a RIO if required, but was stood down after the incident was resolved.
+
+The initial call came from Dyfed Powys Police under reference 333 when it was advised that their report had come from the Ambulance Service (reference 1523673) and the BTP were also made aware by Dyfed Powys Police under their reference 470.",13

test/testCSVConverter.js

@@ -240,4 +240,16 @@ describe("CSV Converter", function() {
     });
     rs.pipe(csvConverter);
   });
+  it ("should process column with linebreaks",function(done){
+    var testData=__dirname+"/data/lineBreak";
+    var rs=fs.createReadStream(testData);
+    var csvConverter=new CSVAdv({
+      constructResult:false
+    });
+    csvConverter.on("record_parsed",function(d){
+      assert(d.Period===13);
+      done();
+    });
+    rs.pipe(csvConverter);
+  });
 });