initial commit.

node_modules/csvtojson/src/CSVError.test.ts

@@ -0,0 +1,6 @@
+import CSVError from "./CSVError";
+import assert from "assert";
+describe("CSVError",()=>{
+  it ("should toString()",()=>{
+  })
+})

node_modules/csvtojson/src/CSVError.ts

@@ -0,0 +1,27 @@
+export default class CSVError extends Error {
+  static column_mismatched(index: number, extra?: string) {
+    return new CSVError("column_mismatched", index, extra);
+  }
+  static unclosed_quote(index: number, extra?: string) {
+    return new CSVError("unclosed_quote", index, extra);
+  }
+  static fromJSON(obj) {
+    return new CSVError(obj.err, obj.line, obj.extra);
+  }
+  constructor(
+    public err: string,
+    public line: number,
+    public extra?: string
+  ) {
+    super("Error: " + err + ". JSON Line number: " + line + (extra ? " near: " + extra : ""));
+    this.name = "CSV Parse Error";
+  }
+  toJSON() {
+    return {
+      err: this.err,
+      line: this.line,
+      extra: this.extra
+    }
+  }
+
+}

node_modules/csvtojson/src/Converter.ts

@@ -0,0 +1,189 @@
+import { Transform, TransformOptions, Readable } from "stream";
+import { CSVParseParam, mergeParams } from "./Parameters";
+import { ParseRuntime, initParseRuntime } from "./ParseRuntime";
+import P from "bluebird";
+import { stringToLines } from "./fileline";
+import { map } from "lodash/map";
+import { RowSplit, RowSplitResult } from "./rowSplit";
+import getEol from "./getEol";
+import lineToJson, { JSONResult } from "./lineToJson";
+import { Processor, ProcessLineResult } from "./Processor";
+// import { ProcessorFork } from "./ProcessFork";
+import { ProcessorLocal } from "./ProcessorLocal";
+import { Result } from "./Result";
+import CSVError from "./CSVError";
+import { bufFromString } from "./util";
+
+
+
+export class Converter extends Transform implements PromiseLike<any[]> {
+  preRawData(onRawData: PreRawDataCallback): Converter {
+    this.runtime.preRawDataHook = onRawData;
+    return this;
+  }
+  preFileLine(onFileLine: PreFileLineCallback): Converter {
+    this.runtime.preFileLineHook = onFileLine;
+    return this;
+  }
+  subscribe(
+    onNext?: (data: any, lineNumber: number) => void | PromiseLike<void>,
+    onError?: (err: CSVError) => void,
+    onCompleted?: () => void): Converter {
+    this.parseRuntime.subscribe = {
+      onNext,
+      onError,
+      onCompleted
+    }
+    return this;
+  }
+  fromFile(filePath: string, options?: string | CreateReadStreamOption | undefined): Converter {
+    const fs = require("fs");
+    // var rs = null;
+    // this.wrapCallback(cb, function () {
+    //   if (rs && rs.destroy) {
+    //     rs.destroy();
+    //   }
+    // });
+    fs.exists(filePath, (exist) => {
+      if (exist) {
+        const rs = fs.createReadStream(filePath, options);
+        rs.pipe(this);
+      } else {
+        this.emit('error', new Error("File does not exist. Check to make sure the file path to your csv is correct."));
+      }
+    });
+    return this;
+  }
+  fromStream(readStream: Readable): Converter {
+    readStream.pipe(this);
+    return this;
+  }
+  fromString(csvString: string): Converter {
+    const csv = csvString.toString();
+    const read = new Readable();
+    let idx = 0;
+    read._read = function (size) {
+      if (idx >= csvString.length) {
+        this.push(null);
+      } else {
+        const str = csvString.substr(idx, size);
+        this.push(str);
+        idx += size;
+      }
+    }
+    return this.fromStream(read);
+  }
+  then<TResult1 = any[], TResult2 = never>(onfulfilled?: (value: any[]) => TResult1 | PromiseLike<TResult1>, onrejected?: (reason: any) => TResult2 | PromiseLike<TResult2>): PromiseLike<TResult1 | TResult2> {
+    return new P((resolve, reject) => {
+      this.parseRuntime.then = {
+        onfulfilled: (value: any[]) => {
+          if (onfulfilled) {
+            resolve(onfulfilled(value));
+          } else {
+            resolve(value as any);
+          }
+        },
+        onrejected: (err: Error) => {
+          if (onrejected) {
+            resolve(onrejected(err));
+          } else {
+            reject(err);
+          }
+        }
+      }
+    });
+  }
+  public get parseParam(): CSVParseParam {
+    return this.params;
+  }
+  public get parseRuntime(): ParseRuntime {
+    return this.runtime;
+  }
+  private params: CSVParseParam;
+  private runtime: ParseRuntime;
+  private processor: Processor;
+  private result: Result;
+  constructor(param?: Partial<CSVParseParam>, public options: TransformOptions = {}) {
+    super(options);
+    this.params = mergeParams(param);
+    this.runtime = initParseRuntime(this);
+    this.result = new Result(this);
+    // if (this.params.fork) {
+    //   this.processor = new ProcessorFork(this);
+    // } else {
+    this.processor = new ProcessorLocal(this);
+    // }
+    this.once("error", (err: any) => {
+      // console.log("BBB");
+      //wait for next cycle to emit the errors.
+      setImmediate(() => {
+        this.result.processError(err);
+        this.emit("done", err);
+      });
+
+    });
+    this.once("done", () => {
+      this.processor.destroy();
+    })
+
+    return this;
+  }
+  _transform(chunk: any, encoding: string, cb: Function) {
+    this.processor.process(chunk)
+      .then((result) => {
+        // console.log(result);
+        if (result.length > 0) {
+          this.runtime.started = true;
+
+          return this.result.processResult(result);
+        }
+      })
+      .then(() => {
+        this.emit("drained");
+        cb();
+      }, (error) => {
+        this.runtime.hasError = true;
+        this.runtime.error = error;
+        this.emit("error", error);
+        cb();
+      });
+  }
+  _flush(cb: Function) {
+    this.processor.flush()
+      .then((data) => {
+        if (data.length > 0) {
+
+          return this.result.processResult(data);
+        }
+      })
+      .then(() => {
+        this.processEnd(cb);
+      }, (err) => {
+        this.emit("error", err);
+        cb();
+      })
+  }
+  private processEnd(cb) {
+    this.result.endProcess();
+    this.emit("done");
+    cb();
+  }
+  get parsedLineNumber(): number {
+    return this.runtime.parsedLineNumber;
+  }
+}
+export interface CreateReadStreamOption {
+  flags?: string;
+  encoding?: string;
+  fd?: number;
+  mode?: number;
+  autoClose?: boolean;
+  start?: number;
+  end?: number;
+  highWaterMark?: number;
+}
+export type CallBack = (err: Error, data: Array<any>) => void;
+
+
+export type PreFileLineCallback = (line: string, lineNumber: number) => string | PromiseLike<string>;
+export type PreRawDataCallback = (csvString: string) => string | PromiseLike<string>;

node_modules/csvtojson/src/Parameters.ts

@@ -0,0 +1,136 @@
+export interface CSVParseParam {
+  /**
+   *   delimiter used for seperating columns. Use "auto" if delimiter is unknown in advance, in this case, delimiter will be auto-detected (by best attempt). Use an array to give a list of potential delimiters e.g. [",","|","$"]. default: ","
+   */
+  delimiter: string | string[];
+  /**
+   *  This parameter instructs the parser to ignore columns as specified by the regular expression. Example: /(name|age)/ will ignore columns whose header contains "name" or "age"
+   */
+  ignoreColumns?: RegExp;
+  /**
+   *  This parameter instructs the parser to include only those columns as specified by the regular expression. Example: /(name|age)/ will parse and include columns whose header contains "name" or "age"
+   */
+  includeColumns?: RegExp;
+  /**
+   *  If a column contains delimiter, it is able to use quote character to surround the column content. e.g. "hello, world" wont be split into two columns while parsing. Set to "off" will ignore all quotes. default: " (double quote)
+   */
+  quote: string;
+  /**
+   *  Indicate if parser trim off spaces surrounding column content. e.g. " content " will be trimmed to "content". Default: true
+   */
+  trim: boolean;
+  /**
+   *  This parameter turns on and off whether check field type. Default is false. 
+   */
+  checkType: boolean;
+  /**
+   *  Ignore the empty value in CSV columns. If a column value is not given, set this to true to skip them. Default: false.
+   */
+  ignoreEmpty: boolean;
+  /**
+   *  Delegate parsing work to another process.
+   */
+  // fork: boolean;
+  /**
+   *  Indicating csv data has no header row and first row is data row. Default is false.
+   */
+  noheader: boolean;
+  /**
+   *  An array to specify the headers of CSV data. If --noheader is false, this value will override CSV header row. Default: null. Example: ["my field","name"].
+   */
+  headers?: string[];
+  /**
+   *  Don't interpret dots (.) and square brackets in header fields as nested object or array identifiers at all (treat them like regular characters for JSON field identifiers). Default: false.
+   */
+  flatKeys: boolean;
+  /**
+   *  the max character a csv row could have. 0 means infinite. If max number exceeded, parser will emit "error" of "row_exceed". if a possibly corrupted csv data provided, give it a number like 65535 so the parser wont consume memory. default: 0
+   */
+  maxRowLength: number;
+  /**
+   *  whether check column number of a row is the same as headers. If column number mismatched headers number, an error of "mismatched_column" will be emitted.. default: false
+   */
+  checkColumn: boolean;
+  /**
+   *  escape character used in quoted column. Default is double quote (") according to RFC4108. Change to back slash (\) or other chars for your own case.
+   */
+  escape: string;
+  /**
+   *   Allows override parsing logic for a specific column. It accepts a JSON object with fields like: headName: <String | Function> . e.g. {field1:'number'} will use built-in number parser to convert value of the field1 column to number. Another example {"name":nameProcessFunc} will use specified function to parse the value.
+   */
+  colParser: {
+    [key: string]: string | CellParser | ColumnParam
+  };
+  /**
+   *  End of line character. If omitted, parser will attempt to retrieve it from the first chunks of CSV data
+   */
+  eol?: string;
+  /**
+   *  Always interpret each line (as defined by eol) as a row. This will prevent eol characters from being used within a row (even inside a quoted field). Default is false. Change to true if you are confident no inline line breaks (like line break in a cell which has multi line text)
+   */
+  alwaysSplitAtEOL: boolean;
+  /**
+   * The format to be converted to. "json" (default) -- convert csv to json. "csv" -- convert csv to csv row array. "line" -- convert csv to csv line string 
+   */
+  output: "json" | "csv" | "line";
+
+  /**
+   * Convert string "null" to null object in JSON outputs. Default is false.
+   */
+  nullObject:boolean;
+  /**
+   * Define the format required by downstream (this parameter does not work if objectMode is on). `line` -- json is emitted in a single line separated by a line breake like "json1\njson2" . `array` -- downstream requires array format like "[json1,json2]". Default is line.
+   */
+  downstreamFormat: "line" | "array";
+  /**
+   * Define whether .then(callback) returns all JSON data in its callback. Default is true. Change to false to save memory if subscribing json lines.
+   */
+  needEmitAll: boolean;
+}
+
+export type CellParser = (item: string, head: string, resultRow: any, row: string[], columnIndex: number) => any;
+
+export interface ColumnParam {
+  flat?: boolean;
+  cellParser?: string | CellParser;
+}
+
+export function mergeParams(params?: Partial<CSVParseParam>): CSVParseParam {
+  const defaultParam: CSVParseParam = {
+    delimiter: ',',
+    ignoreColumns: undefined,
+    includeColumns: undefined,
+    quote: '"',
+    trim: true,
+    checkType: false,
+    ignoreEmpty: false,
+    // fork: false,
+    noheader: false,
+    headers: undefined,
+    flatKeys: false,
+    maxRowLength: 0,
+    checkColumn: false,
+    escape: '"',
+    colParser: {},
+    eol: undefined,
+    alwaysSplitAtEOL: false,
+    output: "json",
+    nullObject: false,
+    downstreamFormat:"line",
+    needEmitAll:true
+  }
+  if (!params) {
+    params = {};
+  }
+  for (let key in params) {
+    if (params.hasOwnProperty(key)) {
+      if (Array.isArray(params[key])) {
+        defaultParam[key] = [].concat(params[key]);
+      } else {
+        defaultParam[key] = params[key];
+      }
+    }
+  }
+  return defaultParam;
+}
+

node_modules/csvtojson/src/ParseRuntime.ts

@@ -0,0 +1,90 @@
+import { CSVParseParam, CellParser } from "./Parameters";
+import { Converter, PreRawDataCallback, PreFileLineCallback } from "./Converter";
+import { ChildProcess } from "child_process";
+import CSVError from "./CSVError";
+
+export interface ParseRuntime {
+  /**
+   * If need convert ignoreColumn from column name(string) to column index (number). Parser needs column index.
+   */
+  needProcessIgnoreColumn: boolean;
+  /**
+   * If need convert includeColumn from column name(string) to column index (number). Parser needs column index.
+   */
+  needProcessIncludeColumn: boolean;
+  /**
+   * the indexes of columns to reserve, undefined means reserve all, [] means hide all
+   */
+  selectedColumns?: number[];
+  ended: boolean;
+  hasError: boolean;
+  error?: Error;
+  /**
+   * Inferred delimiter
+   */
+  delimiter: string | string[];
+  /**
+   * Inferred eol
+   */
+  eol?: string;
+  /**
+   * Converter function for a column. Populated at runtime.
+   */
+  columnConv: (CellParser | null)[],
+  headerType: any[],
+  headerTitle: string[],
+  headerFlag: any[],
+  /**
+   * Inferred headers
+   */
+  headers?: any[],
+  csvLineBuffer?: Buffer,
+  
+  /**
+   * after first chunk of data being processed and emitted, started will become true.
+   */
+  started: boolean,
+  preRawDataHook?: PreRawDataCallback,
+  preFileLineHook?: PreFileLineCallback,
+  parsedLineNumber: number,
+
+  columnValueSetter: Function[];
+  subscribe?: {
+    onNext?: (data: any, lineNumber:number) => void | PromiseLike<void>;
+    onError?: (err: CSVError) => void;
+    onCompleted?: () => void;
+  };
+  then?: {
+    onfulfilled: (value: any[]) => any;
+    onrejected: (err: Error) => any;
+  }
+
+}
+export function initParseRuntime(converter: Converter): ParseRuntime {
+  const params = converter.parseParam;
+  const rtn: ParseRuntime = {
+    needProcessIgnoreColumn: false,
+    needProcessIncludeColumn: false,
+    selectedColumns: undefined,
+    ended: false,
+    hasError: false,
+    error: undefined,
+    delimiter: converter.parseParam.delimiter,
+    eol: converter.parseParam.eol,
+    columnConv: [],
+    headerType: [],
+    headerTitle: [],
+    headerFlag: [],
+    headers: undefined,
+    started: false,
+    parsedLineNumber: 0,
+    columnValueSetter: [],
+  }
+  if (params.ignoreColumns) {
+    rtn.needProcessIgnoreColumn = true;
+  }
+  if (params.includeColumns) {
+    rtn.needProcessIncludeColumn = true;
+  }
+  return rtn;
+}

node_modules/csvtojson/src/ProcessFork.ts

@@ -0,0 +1,150 @@
+import { Processor, ProcessLineResult } from "./Processor";
+import P from "bluebird"
+import { Converter } from "./Converter";
+import { ChildProcess } from "child_process";
+import { CSVParseParam, mergeParams } from "./Parameters";
+import { ParseRuntime } from "./ParseRuntime";
+import { Readable, Writable } from "stream";
+import { bufFromString, emptyBuffer } from "./util";
+import CSVError from "./CSVError";
+
+export class ProcessorFork extends Processor {
+  flush(): P<ProcessLineResult[]> {
+    return new P((resolve, reject) => {
+      // console.log("flush");
+      this.finalChunk = true;
+      this.next = resolve;
+      this.childProcess.stdin.end();
+      // this.childProcess.stdout.on("end",()=>{
+      //   // console.log("!!!!");
+      //   this.flushResult();
+      // })
+    });
+  }
+  destroy(): P<void> {
+    this.childProcess.kill();
+    return P.resolve();
+  }
+  childProcess: ChildProcess;
+  inited: boolean = false;
+  private resultBuf: ProcessLineResult[] = [];
+  private leftChunk: string = "";
+  private finalChunk: boolean = false;
+  private next?: (result: ProcessLineResult[]) => any;
+  constructor(protected converter: Converter) {
+    super(converter);
+    this.childProcess = require("child_process").spawn(process.execPath, [__dirname + "/../v2/worker.js"], {
+      stdio: ["pipe", "pipe", "pipe", "ipc"]
+    });
+    this.initWorker();
+  }
+  private prepareParam(param:CSVParseParam):any{
+    const clone:any=mergeParams(param);
+    if (clone.ignoreColumns){
+      clone.ignoreColumns={
+        source:clone.ignoreColumns.source,
+        flags:clone.ignoreColumns.flags
+      }
+    }
+    if (clone.includeColumns){
+      clone.includeColumns={
+        source:clone.includeColumns.source,
+        flags:clone.includeColumns.flags
+      }
+    }
+    return clone;
+  }
+  private initWorker() {
+    this.childProcess.on("exit",()=>{
+      this.flushResult();
+    })
+    this.childProcess.send({
+      cmd: "init",
+      params: this.prepareParam(this.converter.parseParam)
+    } as InitMessage);
+    this.childProcess.on("message", (msg: Message) => {
+      if (msg.cmd === "inited") {
+        this.inited = true;
+      } else if (msg.cmd === "eol") {
+        if (this.converter.listeners("eol").length > 0){
+          this.converter.emit("eol",(msg as StringMessage).value);
+        }
+      }else if (msg.cmd === "header") {
+        if (this.converter.listeners("header").length > 0){
+          this.converter.emit("header",(msg as StringMessage).value);
+        }
+      }else if (msg.cmd === "done"){
+
+        // this.flushResult();
+      }
+
+    });
+    this.childProcess.stdout.on("data", (data) => {
+      // console.log("stdout", data.toString());
+      const res = data.toString();
+      // console.log(res);
+      this.appendBuf(res);
+
+    });
+    this.childProcess.stderr.on("data", (data) => {
+      // console.log("stderr", data.toString());
+      this.converter.emit("error", CSVError.fromJSON(JSON.parse(data.toString())));
+    });
+
+  }
+  private flushResult() {
+    // console.log("flush result", this.resultBuf.length);
+    if (this.next) {
+      this.next(this.resultBuf);
+    }
+    this.resultBuf = [];
+  }
+  private appendBuf(data: string) {
+    const res = this.leftChunk + data;
+    const list = res.split("\n");
+    let counter = 0;
+    const lastBit = list[list.length - 1];
+    if (lastBit !== "") {
+      this.leftChunk = list.pop() || "";
+    } else {
+      this.leftChunk = "";
+    }
+    this.resultBuf=this.resultBuf.concat(list);
+    // while (list.length) {
+    //   let item = list.shift() || "";
+    //   if (item.length === 0 ) {
+    //     continue;
+    //   }
+    //   // if (this.params.output !== "line") {
+    //   //     item = JSON.parse(item);
+    //   // }
+    //   this.resultBuf.push(item);
+    //   counter++;
+    // }
+    // console.log("buf length",this.resultBuf.length);
+  }
+
+  process(chunk: Buffer): P<ProcessLineResult[]> {
+    return new P((resolve, reject) => {
+      // console.log("chunk", chunk.length);
+      this.next = resolve;
+      // this.appendReadBuf(chunk);
+      this.childProcess.stdin.write(chunk, () => {
+        // console.log("chunk callback");
+        this.flushResult();
+      });
+    });
+  }
+}
+
+export interface Message {
+  cmd: string
+}
+
+export interface InitMessage extends Message {
+  params: any;
+}
+export interface StringMessage extends Message {
+  value: string
+}
+export const EOM = "\x03";

node_modules/csvtojson/src/Processor.ts

@@ -0,0 +1,18 @@
+import { Converter } from "./Converter";
+import P from "bluebird";
+import { JSONResult } from "./lineToJson";
+import { CSVParseParam } from "./Parameters";
+import { ParseRuntime } from "./ParseRuntime";
+
+export abstract class Processor {
+  protected params: CSVParseParam;
+  protected runtime: ParseRuntime;
+  constructor(protected converter: Converter) {
+    this.params = converter.parseParam;
+    this.runtime = converter.parseRuntime;
+  }
+  abstract process(chunk: Buffer,finalChunk?:boolean): P<ProcessLineResult[]>
+  abstract destroy():P<void>;
+  abstract flush(): P<ProcessLineResult[]>;
+}
+export type ProcessLineResult = string | string[] | JSONResult;

node_modules/csvtojson/src/ProcessorLocal.test.ts

@@ -0,0 +1,40 @@
+import {ProcessorLocal} from "./ProcessorLocal";
+import { Converter } from "./Converter";
+import P from "bluebird";
+import {readFileSync} from "fs";
+import path from "path";
+import assert from "assert";
+import { JSONResult } from "./lineToJson";
+const dataDir=path.join(__dirname,"../test/data/");
+describe("ProcessLocal",()=>{
+  it ("should process csv chunks and output json",async function (){
+    const processor=new ProcessorLocal(new Converter());
+    const data=readFileSync(dataDir+"/complexJSONCSV");
+    const lines=await processor.process(data);
+    assert(lines.length === 2);
+    const line0=lines[0] as JSONResult;
+    assert.equal(line0.fieldA.title,"Food Factory");
+    assert.equal(line0.fieldA.children.length,2);
+    assert.equal(line0.fieldA.children[1].employee[0].name,"Tim");
+  })
+  it ("should process csv chunks and output csv rows",async function (){
+    const processor=new ProcessorLocal(new Converter({output:"line"}));
+    const data=readFileSync(dataDir+"/complexJSONCSV");
+    const lines=await processor.process(data);
+    
+    assert(lines.length === 2);
+  })
+  it ("should return empty array if preRawHook removed the data",()=>{
+    const conv=new Converter();
+    conv.preRawData((str)=>{
+      return "";
+    });
+    const processor=new ProcessorLocal(conv);
+    const data=readFileSync(dataDir+"/complexJSONCSV");
+    return processor.process(data)
+    .then((list)=>{
+      assert.equal(list.length,0);
+    })
+  })
+})
+

node_modules/csvtojson/src/ProcessorLocal.ts

@@ -0,0 +1,270 @@
+import { Processor, ProcessLineResult } from "./Processor";
+import P from "bluebird";
+import { prepareData } from "./dataClean";
+import getEol from "./getEol";
+import { stringToLines } from "./fileline";
+import { bufFromString, filterArray,trimLeft } from "./util";
+import { RowSplit } from "./rowSplit";
+import lineToJson from "./lineToJson";
+import { ParseRuntime } from "./ParseRuntime";
+import CSVError from "./CSVError";
+
+
+
+export class ProcessorLocal extends Processor {
+  flush(): P<ProcessLineResult[]> {
+    if (this.runtime.csvLineBuffer && this.runtime.csvLineBuffer.length > 0) {
+      const buf = this.runtime.csvLineBuffer;
+      this.runtime.csvLineBuffer = undefined;
+      return this.process(buf, true)
+        .then((res) => {
+          if (this.runtime.csvLineBuffer && this.runtime.csvLineBuffer.length > 0) {
+            return P.reject(CSVError.unclosed_quote(this.runtime.parsedLineNumber, this.runtime.csvLineBuffer.toString()))
+          } else {
+            return P.resolve(res);
+          }
+        })
+    } else {
+      return P.resolve([]);
+    }
+  }
+  destroy(): P<void> {
+    return P.resolve();
+  }
+  private rowSplit: RowSplit = new RowSplit(this.converter);
+  private eolEmitted = false;
+  private _needEmitEol?: boolean = undefined;
+  private get needEmitEol() {
+    if (this._needEmitEol === undefined) {
+      this._needEmitEol = this.converter.listeners("eol").length > 0;
+    }
+    return this._needEmitEol;
+  }
+  private headEmitted = false;
+  private _needEmitHead?: boolean = undefined;
+  private get needEmitHead() {
+    if (this._needEmitHead === undefined) {
+      this._needEmitHead = this.converter.listeners("header").length > 0;
+    }
+    return this._needEmitHead;
+
+  }
+  process(chunk: Buffer, finalChunk = false): P<ProcessLineResult[]> {
+    let csvString: string;
+    if (finalChunk) {
+      csvString = chunk.toString();
+    } else {
+      csvString = prepareData(chunk, this.converter.parseRuntime);
+
+    }
+    return P.resolve()
+      .then(() => {
+        if (this.runtime.preRawDataHook) {
+          return this.runtime.preRawDataHook(csvString);
+        } else {
+          return csvString;
+        }
+      })
+      .then((csv) => {
+        if (csv && csv.length > 0) {
+          return this.processCSV(csv, finalChunk);
+        } else {
+          return P.resolve([]);
+        }
+      })
+  }
+  private processCSV(csv: string, finalChunk: boolean): P<ProcessLineResult[]> {
+    const params = this.params;
+    const runtime = this.runtime;
+    if (!runtime.eol) {
+      getEol(csv, runtime);
+    }
+    if (this.needEmitEol && !this.eolEmitted && runtime.eol) {
+      this.converter.emit("eol", runtime.eol);
+      this.eolEmitted = true;
+    }
+    // trim csv file has initial blank lines.
+    if (params.ignoreEmpty && !runtime.started) {
+      csv = trimLeft(csv);
+    }
+    const stringToLineResult = stringToLines(csv, runtime);
+    if (!finalChunk) {
+      this.prependLeftBuf(bufFromString(stringToLineResult.partial));
+    } else {
+      stringToLineResult.lines.push(stringToLineResult.partial);
+      stringToLineResult.partial = "";
+    }
+    if (stringToLineResult.lines.length > 0) {
+      let prom: P<string[]>;
+      if (runtime.preFileLineHook) {
+        prom = this.runPreLineHook(stringToLineResult.lines);
+      } else {
+        prom = P.resolve(stringToLineResult.lines);
+      }
+      return prom.then((lines) => {
+        if (!runtime.started
+          && !this.runtime.headers
+        ) {
+          return this.processDataWithHead(lines);
+        } else {
+          return this.processCSVBody(lines);
+        }
+
+      })
+
+    } else {
+
+      return P.resolve([]);
+    }
+
+  }
+  private processDataWithHead(lines: string[]): ProcessLineResult[] {
+    if (this.params.noheader) {
+      if (this.params.headers) {
+        this.runtime.headers = this.params.headers;
+      } else {
+        this.runtime.headers = [];
+      }
+    } else {
+      let left = "";
+      let headerRow: string[] = [];
+      while (lines.length) {
+        const line = left + lines.shift();
+        const row = this.rowSplit.parse(line);
+        if (row.closed) {
+          headerRow = row.cells;
+          left = "";
+          break;
+        } else {
+          left = line + getEol(line, this.runtime);
+        }
+      }
+      this.prependLeftBuf(bufFromString(left));
+
+      if (headerRow.length === 0) {
+        return [];
+      }
+      if (this.params.headers) {
+        this.runtime.headers = this.params.headers;
+      } else {
+        this.runtime.headers = headerRow;
+      }
+    }
+    if (this.runtime.needProcessIgnoreColumn || this.runtime.needProcessIncludeColumn) {
+      this.filterHeader();
+    }
+    if (this.needEmitHead && !this.headEmitted) {
+      this.converter.emit("header", this.runtime.headers);
+      this.headEmitted = true;
+    }
+    return this.processCSVBody(lines);
+  }
+  private filterHeader() {
+    this.runtime.selectedColumns = [];
+    if (this.runtime.headers) {
+      const headers = this.runtime.headers;
+      for (let i = 0; i < headers.length; i++) {
+        if (this.params.ignoreColumns) {
+          if (this.params.ignoreColumns.test(headers[i])) {
+            if (this.params.includeColumns && this.params.includeColumns.test(headers[i])) {
+              this.runtime.selectedColumns.push(i);
+            } else {
+              continue;
+            }
+          } else {
+            this.runtime.selectedColumns.push(i);
+          }
+        } else if (this.params.includeColumns) {
+          if (this.params.includeColumns.test(headers[i])) {
+            this.runtime.selectedColumns.push(i);
+          }
+        } else {
+          this.runtime.selectedColumns.push(i);
+        }
+        // if (this.params.includeColumns && this.params.includeColumns.test(headers[i])){
+        //   this.runtime.selectedColumns.push(i);
+        // }else{
+        //   if (this.params.ignoreColumns && this.params.ignoreColumns.test(headers[i])){
+        //     continue;
+        //   }else{
+        //     if (this.params.ignoreColumns && !this.params.includeColumns){
+        //       this.runtime.selectedColumns.push(i);
+        //     }
+
+        //   }
+        // }
+      }
+      this.runtime.headers = filterArray(this.runtime.headers, this.runtime.selectedColumns);
+    }
+
+  }
+  private processCSVBody(lines: string[]): ProcessLineResult[] {
+    if (this.params.output === "line") {
+      return lines;
+    } else {
+      const result = this.rowSplit.parseMultiLines(lines);
+      this.prependLeftBuf(bufFromString(result.partial));
+      if (this.params.output === "csv") {
+        return result.rowsCells;
+      } else {
+        return lineToJson(result.rowsCells, this.converter);
+      }
+    }
+
+    // var jsonArr = linesToJson(lines.lines, params, this.recordNum);
+    // this.processResult(jsonArr);
+    // this.lastIndex += jsonArr.length;
+    // this.recordNum += jsonArr.length;
+  }
+
+  private prependLeftBuf(buf: Buffer) {
+    if (buf) {
+      if (this.runtime.csvLineBuffer) {
+        this.runtime.csvLineBuffer = Buffer.concat([buf, this.runtime.csvLineBuffer]);
+      } else {
+        this.runtime.csvLineBuffer = buf;
+      }
+    }
+
+  }
+  private runPreLineHook(lines: string[]): P<string[]> {
+    return new P((resolve, reject) => {
+      processLineHook(lines, this.runtime, 0, (err) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(lines);
+        }
+      })
+    });
+  }
+}
+
+function processLineHook(lines: string[], runtime: ParseRuntime, offset: number,
+  cb: (err?) => void
+) {
+  if (offset >= lines.length) {
+    cb();
+  } else {
+    if (runtime.preFileLineHook) {
+      const line = lines[offset];
+      const res = runtime.preFileLineHook(line, runtime.parsedLineNumber + offset);
+      offset++;
+      if (res && (res as PromiseLike<string>).then) {
+        (res as PromiseLike<string>).then((value) => {
+          lines[offset - 1] = value;
+          processLineHook(lines, runtime, offset, cb);
+        });
+      } else {
+        lines[offset - 1] = res as string;
+        while (offset < lines.length) {
+          lines[offset] = runtime.preFileLineHook(lines[offset], runtime.parsedLineNumber + offset) as string;
+          offset++;
+        }
+        cb();
+      }
+    } else {
+      cb();
+    }
+  }
+}

node_modules/csvtojson/src/Result.test.ts

@@ -0,0 +1,22 @@
+import {Result} from "./Result";
+import { Converter } from "./Converter";
+import P from "bluebird";
+import {readFileSync} from "fs";
+import path from "path";
+import assert from "assert";
+import { JSONResult } from "./lineToJson";
+const dataDir=path.join(__dirname,"../test/data/");
+
+describe("Result",()=>{
+  it ("should return need push downstream based on needEmitAll parameter",function (){
+      const conv=new Converter();
+      const res=new Result(conv);
+      assert.equal(res["needEmitAll"],false);
+      conv.then();
+      assert.equal(res["needEmitAll"],true);
+      conv.parseParam.needEmitAll=false;
+      assert.equal(res["needEmitAll"],false);
+  });
+
+})
+

node_modules/csvtojson/src/Result.ts

@@ -0,0 +1,164 @@
+import { Converter } from "./Converter";
+import { ProcessLineResult } from "./Processor";
+import P from "bluebird";
+import CSVError from "./CSVError";
+import { EOL } from "os";
+export class Result {
+  private get needEmitLine(): boolean {
+    return !!this.converter.parseRuntime.subscribe && !!this.converter.parseRuntime.subscribe.onNext || this.needPushDownstream
+  }
+  private _needPushDownstream?: boolean;
+  private get needPushDownstream(): boolean {
+    if (this._needPushDownstream === undefined) {
+      this._needPushDownstream = this.converter.listeners("data").length > 0 || this.converter.listeners("readable").length > 0;
+    }
+    return this._needPushDownstream;
+  }
+  private get needEmitAll(): boolean {
+    return !!this.converter.parseRuntime.then && this.converter.parseParam.needEmitAll;
+    // return !!this.converter.parseRuntime.then;
+  }
+  private finalResult: any[] = [];
+  constructor(private converter: Converter) { }
+  processResult(resultLines: ProcessLineResult[]): P<any> {
+    const startPos = this.converter.parseRuntime.parsedLineNumber;
+    if (this.needPushDownstream && this.converter.parseParam.downstreamFormat === "array") {
+      if (startPos === 0) {
+        pushDownstream(this.converter, "[" + EOL);
+      }
+    }
+    // let prom: P<any>;
+    return new P((resolve, reject) => {
+      if (this.needEmitLine) {
+        processLineByLine(
+          resultLines,
+          this.converter,
+          0,
+          this.needPushDownstream,
+          (err) => {
+            if (err) {
+              reject(err);
+            } else {
+              this.appendFinalResult(resultLines);
+              resolve();
+            }
+          },
+        )
+        // resolve();
+      } else {
+        this.appendFinalResult(resultLines);
+        resolve();
+      }
+    })
+  }
+  appendFinalResult(lines: any[]) {
+    if (this.needEmitAll) {
+      this.finalResult = this.finalResult.concat(lines);
+    }
+    this.converter.parseRuntime.parsedLineNumber += lines.length;
+  }
+  processError(err: CSVError) {
+    if (this.converter.parseRuntime.subscribe && this.converter.parseRuntime.subscribe.onError) {
+      this.converter.parseRuntime.subscribe.onError(err);
+    }
+    if (this.converter.parseRuntime.then && this.converter.parseRuntime.then.onrejected) {
+      this.converter.parseRuntime.then.onrejected(err);
+    }
+  }
+  endProcess() {
+    
+      if (this.converter.parseRuntime.then && this.converter.parseRuntime.then.onfulfilled) {
+        if (this.needEmitAll) {
+          this.converter.parseRuntime.then.onfulfilled(this.finalResult);
+        }else{
+          this.converter.parseRuntime.then.onfulfilled([]);
+        }
+      }
+    if (this.converter.parseRuntime.subscribe && this.converter.parseRuntime.subscribe.onCompleted) {
+      this.converter.parseRuntime.subscribe.onCompleted();
+    }
+    if (this.needPushDownstream && this.converter.parseParam.downstreamFormat === "array") {
+      pushDownstream(this.converter, "]" + EOL);
+    }
+  }
+}
+
+function processLineByLine(
+  lines: ProcessLineResult[],
+
+  conv: Converter,
+  offset: number,
+  needPushDownstream: boolean,
+  cb: (err?) => void,
+) {
+  if (offset >= lines.length) {
+    cb();
+  } else {
+    if (conv.parseRuntime.subscribe && conv.parseRuntime.subscribe.onNext) {
+      const hook = conv.parseRuntime.subscribe.onNext;
+      const nextLine = lines[offset];
+      const res = hook(nextLine, conv.parseRuntime.parsedLineNumber + offset);
+      offset++;
+      // if (isAsync === undefined) {
+      if (res && res.then) {
+        res.then(function () {
+          processRecursive(lines, hook, conv, offset, needPushDownstream, cb, nextLine);
+        }, cb);
+      } else {
+        // processRecursive(lines, hook, conv, offset, needPushDownstream, cb, nextLine, false);
+        if (needPushDownstream) {
+          pushDownstream(conv, nextLine);
+        }
+        while (offset < lines.length) {
+          const line = lines[offset];
+          hook(line, conv.parseRuntime.parsedLineNumber + offset);
+          offset++;
+          if (needPushDownstream) {
+            pushDownstream(conv, line);
+          }
+        }
+        cb();
+      }
+      // } else if (isAsync === true) {
+      //   (res as PromiseLike<void>).then(function () {
+      //     processRecursive(lines, hook, conv, offset, needPushDownstream, cb, nextLine, true);
+      //   }, cb);
+      // } else if (isAsync === false) {
+      //   processRecursive(lines, hook, conv, offset, needPushDownstream, cb, nextLine, false);
+      // }
+    } else {
+      if (needPushDownstream) {
+        while (offset < lines.length) {
+          const line = lines[offset++];
+          pushDownstream(conv, line);
+        }
+
+      }
+      cb();
+    }
+
+  }
+}
+
+function processRecursive(
+  lines: ProcessLineResult[],
+  hook: (data: any, lineNumber: number) => void | PromiseLike<void>,
+  conv: Converter,
+  offset: number,
+  needPushDownstream: boolean,
+  cb: (err?) => void,
+  res: ProcessLineResult,
+) {
+  if (needPushDownstream) {
+    pushDownstream(conv, res);
+  }
+  processLineByLine(lines, conv, offset, needPushDownstream, cb);
+}
+function pushDownstream(conv: Converter, res: ProcessLineResult) {
+  if (typeof res === "object" && !conv.options.objectMode) {
+    const data = JSON.stringify(res);
+    conv.push(data + (conv.parseParam.downstreamFormat === "array" ? "," + EOL : EOL), "utf8");
+  } else {
+    conv.push(res);
+  }
+}

node_modules/csvtojson/src/dataClean.ts

@@ -0,0 +1,63 @@
+import { ParseRuntime } from "./ParseRuntime";
+import stripBom from "strip-bom";
+/**
+ * For each data chunk coming to parser:
+ * 1. append the data to the buffer that is left from last chunk
+ * 2. check if utf8 chars being split, if does, stripe the bytes and add to left buffer.
+ * 3. stripBom 
+ */
+export function prepareData(chunk: Buffer, runtime: ParseRuntime): string {
+  const workChunk = concatLeftChunk(chunk, runtime);
+  runtime.csvLineBuffer = undefined;
+  const cleanCSVString = cleanUtf8Split(workChunk, runtime).toString("utf8");
+  if (runtime.started === false) {
+    return stripBom(cleanCSVString);
+  } else {
+    return cleanCSVString;
+  }
+}
+/**
+ *  append data to buffer that is left form last chunk
+ */
+function concatLeftChunk(chunk: Buffer, runtime: ParseRuntime): Buffer {
+  if (runtime.csvLineBuffer && runtime.csvLineBuffer.length > 0) {
+    return Buffer.concat([runtime.csvLineBuffer, chunk]);
+  } else {
+    return chunk;
+  }
+}
+/**
+ * check if utf8 chars being split, if does, stripe the bytes and add to left buffer.
+ */
+function cleanUtf8Split(chunk: Buffer, runtime: ParseRuntime): Buffer {
+  let idx = chunk.length - 1;
+  /**
+   * From Keyang:
+   * The code below is to check if a single utf8 char (which could be multiple bytes) being split.
+   * If the char being split, the buffer from two chunk needs to be concat
+   * check how utf8 being encoded to understand the code below. 
+   * If anyone has any better way to do this, please let me know.
+   */
+  if ((chunk[idx] & 1 << 7) != 0) {
+    while ((chunk[idx] & 3 << 6) === 128) {
+      idx--;
+    }
+    idx--;
+  }
+  if (idx != chunk.length - 1) {
+    runtime.csvLineBuffer = chunk.slice(idx + 1);
+    return chunk.slice(0, idx + 1)
+    // var _cb=cb;
+    // var self=this;
+    // cb=function(){
+    //   if (self._csvLineBuffer){
+    //     self._csvLineBuffer=Buffer.concat([bufFromString(self._csvLineBuffer,"utf8"),left]);
+    //   }else{
+    //     self._csvLineBuffer=left;
+    //   }
+    //   _cb();
+    // }
+  } else {
+    return chunk;
+  }
+}

node_modules/csvtojson/src/fileline.test.ts

@@ -0,0 +1,14 @@
+import {stringToLines} from "./fileline";
+import { mergeParams } from "./Parameters";
+import { Converter } from "./Converter";
+var assert = require("assert");
+describe("fileline function", function() {
+  it ("should convert data to multiple lines ", function() {
+    const conv=new Converter();
+    var data = "abcde\nefef";
+    var result = stringToLines(data, conv.parseRuntime);
+    assert.equal(result.lines.length, 1);
+    assert.equal(result.partial, "efef");
+    assert.equal(result.lines[0], "abcde");
+  });
+});

node_modules/csvtojson/src/fileline.ts

@@ -0,0 +1,25 @@
+import { ParseRuntime } from "./ParseRuntime";
+import getEol from "./getEol";
+// const getEol = require("./getEol");
+/**
+ * convert data chunk to file lines array
+ * @param  {string} data  data chunk as utf8 string
+ * @param  {object} param Converter param object
+ * @return {Object}   {lines:[line1,line2...],partial:String}
+ */
+export function stringToLines(data: string, param: ParseRuntime): StringToLinesResult {
+  const eol = getEol(data, param);
+  const lines = data.split(eol);
+  const partial = lines.pop() || "";
+  return { lines: lines, partial: partial };
+};
+
+
+export interface StringToLinesResult {
+  lines: Fileline[],
+  /**
+   * last line which could be incomplete line.
+   */
+  partial: string
+}
+export type Fileline = string;

node_modules/csvtojson/src/getEol.ts

@@ -0,0 +1,21 @@
+import { ParseRuntime } from "./ParseRuntime";
+//return first eol found from a data chunk.
+export default function (data: string, param: ParseRuntime): string {
+  if (!param.eol && data) {
+    for (var i = 0, len = data.length; i < len; i++) {
+      if (data[i] === "\r") {
+        if (data[i + 1] === "\n") {
+          param.eol = "\r\n";
+          break;
+        } else if (data[i + 1]) {
+          param.eol = "\r";
+          break;
+        } 
+      } else if (data[i] === "\n") {
+        param.eol = "\n";
+        break;
+      }
+    }
+  }
+  return param.eol || "\n";
+};

node_modules/csvtojson/src/index.ts

@@ -0,0 +1,10 @@
+import { TransformOptions } from "stream";
+import { CSVParseParam } from "./Parameters";
+import { Converter } from "./Converter";
+
+const helper = function (param?: Partial<CSVParseParam>, options?: TransformOptions): Converter {
+  return new Converter(param, options);
+}
+helper["csv"] = helper;
+helper["Converter"] = Converter;
+export =helper;

node_modules/csvtojson/src/lineToJson.ts

@@ -0,0 +1,210 @@
+import { Converter } from "./Converter";
+import CSVError from "./CSVError";
+import { CellParser, ColumnParam } from "./Parameters";
+import set from "lodash/set";
+import { ParseRuntime } from "./ParseRuntime";
+
+var numReg = /^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$/;
+
+export default function (csvRows: string[][], conv: Converter): JSONResult[] {
+  const res: JSONResult[] = [];
+  for (let i = 0, len = csvRows.length; i < len; i++) {
+    const r = processRow(csvRows[i], conv, i);
+    if (r) {
+      res.push(r);
+    }
+  }
+  return res;
+};
+export type JSONResult = {
+  [key: string]: any
+}
+
+function processRow(row: string[], conv: Converter, index): JSONResult | null {
+
+  if (conv.parseParam.checkColumn && conv.parseRuntime.headers && row.length !== conv.parseRuntime.headers.length) {
+    throw (CSVError.column_mismatched(conv.parseRuntime.parsedLineNumber + index))
+  }
+
+  const headRow = conv.parseRuntime.headers || [];
+  const resultRow = convertRowToJson(row, headRow, conv);
+  if (resultRow) {
+    return resultRow;
+  } else {
+    return null;
+  }
+}
+
+function convertRowToJson(row: string[], headRow: string[], conv: Converter): { [key: string]: any } | null {
+  let hasValue = false;
+  const resultRow = {};
+  
+  for (let i = 0, len = row.length; i < len; i++) {
+    let item = row[i];
+
+    if (conv.parseParam.ignoreEmpty && item === '') {
+      continue;
+    }
+    hasValue = true;
+
+    let head = headRow[i];
+    if (!head || head === "") {
+      head = headRow[i] = "field" + (i + 1);
+    }
+    const convFunc = getConvFunc(head, i, conv);
+    if (convFunc) {
+      const convRes = convFunc(item, head, resultRow, row, i);
+      if (convRes !== undefined) {
+        setPath(resultRow, head, convRes, conv,i);
+      }
+    } else {
+      // var flag = getFlag(head, i, param);
+      // if (flag === 'omit') {
+      //   continue;
+      // }
+      if (conv.parseParam.checkType) {
+        const convertFunc = checkType(item, head, i, conv);
+        item = convertFunc(item);
+      }
+      if (item !== undefined) {
+        setPath(resultRow, head, item, conv,i);
+      }
+    }
+  }
+  if (hasValue) {
+    return resultRow;
+  } else {
+    return null;
+  }
+}
+
+const builtInConv: { [key: string]: CellParser } = {
+  "string": stringType,
+  "number": numberType,
+  "omit": function () { }
+}
+function getConvFunc(head: string, i: number, conv: Converter): CellParser | null {
+  if (conv.parseRuntime.columnConv[i] !== undefined) {
+    return conv.parseRuntime.columnConv[i];
+  } else {
+    let flag = conv.parseParam.colParser[head];
+    if (flag === undefined) {
+      return conv.parseRuntime.columnConv[i] = null;
+    }
+    if (typeof flag === "object") {
+      flag = (flag as ColumnParam).cellParser || "string";
+    }
+    if (typeof flag === "string") {
+      flag = flag.trim().toLowerCase();
+      const builtInFunc = builtInConv[flag];
+      if (builtInFunc) {
+        return conv.parseRuntime.columnConv[i] = builtInFunc;
+      } else {
+        return conv.parseRuntime.columnConv[i] = null;
+      }
+    } else if (typeof flag === "function") {
+      return conv.parseRuntime.columnConv[i] = flag;
+    } else {
+      return conv.parseRuntime.columnConv[i] = null;
+    }
+  }
+}
+function setPath(resultJson: any, head: string, value: any, conv: Converter,headIdx:number) {
+  if (!conv.parseRuntime.columnValueSetter[headIdx]) {
+    if (conv.parseParam.flatKeys) {
+      conv.parseRuntime.columnValueSetter[headIdx] = flatSetter;
+    } else {
+      
+      if (head.indexOf(".") > -1) {
+        const headArr=head.split(".");
+        let jsonHead=true;
+        while(headArr.length>0){
+          const headCom=headArr.shift();
+          if (headCom!.length===0){
+            jsonHead=false;
+            break;
+          }
+        }
+        if (!jsonHead || conv.parseParam.colParser[head] && (conv.parseParam.colParser[head] as ColumnParam).flat) {
+          conv.parseRuntime.columnValueSetter[headIdx] = flatSetter;
+        } else {
+          conv.parseRuntime.columnValueSetter[headIdx] = jsonSetter;
+        }
+      } else {
+        conv.parseRuntime.columnValueSetter[headIdx] = flatSetter;
+      }
+    }
+  }
+  if (conv.parseParam.nullObject ===true && value ==="null"){
+    value=null;
+  }
+  conv.parseRuntime.columnValueSetter[headIdx](resultJson, head, value);
+  // flatSetter(resultJson, head, value);
+
+}
+function flatSetter(resultJson: any, head: string, value: any) {
+  resultJson[head] = value;
+}
+function jsonSetter(resultJson: any, head: string, value: any) {
+  set(resultJson, head, value);
+}
+
+
+function checkType(item: string, head: string, headIdx: number, conv: Converter): Function {
+  if (conv.parseRuntime.headerType[headIdx]) {
+    return conv.parseRuntime.headerType[headIdx];
+  } else if (head.indexOf('number#!') > -1) {
+    return conv.parseRuntime.headerType[headIdx] = numberType;
+  } else if (head.indexOf('string#!') > -1) {
+    return conv.parseRuntime.headerType[headIdx] = stringType;
+  } else if (conv.parseParam.checkType) {
+    return conv.parseRuntime.headerType[headIdx] = dynamicType;
+  } else {
+    return conv.parseRuntime.headerType[headIdx] = stringType;
+  }
+}
+
+function numberType(item) {
+  var rtn = parseFloat(item);
+  if (isNaN(rtn)) {
+    return item;
+  }
+  return rtn;
+}
+
+function stringType(item: string): string {
+  return item.toString();
+}
+
+function dynamicType(item) {
+  var trimed = item.trim();
+  if (trimed === "") {
+    return stringType(item);
+  }
+  if (numReg.test(trimed)) {
+    return numberType(item);
+  } else if (trimed.length === 5 && trimed.toLowerCase() === "false" || trimed.length === 4 && trimed.toLowerCase() === "true") {
+    return booleanType(item);
+  } else if (trimed[0] === "{" && trimed[trimed.length - 1] === "}" || trimed[0] === "[" && trimed[trimed.length - 1] === "]") {
+    return jsonType(item);
+  } else {
+    return stringType(item);
+  }
+}
+
+function booleanType(item) {
+  var trimed = item.trim();
+  if (trimed.length === 5 && trimed.toLowerCase() === "false") {
+    return false;
+  } else {
+    return true;
+  }
+}
+
+function jsonType(item) {
+  try {
+    return JSON.parse(item);
+  } catch (e) {
+    return item;
+  }
+}

node_modules/csvtojson/src/rowSplit.test.ts

@@ -0,0 +1,142 @@
+import { RowSplit, MultipleRowResult, RowSplitResult } from "./rowSplit";
+import { Converter } from "./Converter";
+const assert = require("assert");
+
+describe("Test delimiters", function () {
+  const getDelimiter = (str, opt: { delimiter: string | string[] }): string => {
+    return RowSplit.prototype["getDelimiter"].call({
+      conv: {
+        parseParam: {
+          delimiter: opt.delimiter
+        }
+      }
+    }, str);
+  }
+
+  it("should return the explicitly specified delimiter", function () {
+    var delimiter = ";";
+    var rowStr = "a;b;c";
+    var returnedDelimiter = getDelimiter(rowStr, { delimiter: ";" });
+    assert.equal(returnedDelimiter, delimiter);
+  });
+
+  it("should return the autodetected delimiter if 'auto' specified", function () {
+    var rowStr = "a;b;c";
+    var returnedDelimiter = getDelimiter(rowStr, { delimiter: "auto" });
+    assert(returnedDelimiter === ";");
+  });
+
+  it("should return the ',' delimiter if delimiter cannot be specified, in case of 'auto'", function () {
+    var rowStr = "abc";
+    var returnedDelimiter = getDelimiter(rowStr, { delimiter: "auto" });
+    assert(returnedDelimiter === ",");
+  });
+
+  it("should accetp an array with potential delimiters", function () {
+    var rowStr = "a$b$c";
+    var returnedDelimiter = getDelimiter(rowStr, { delimiter: [",", ";", "$"] });
+    assert(returnedDelimiter === '$');
+  });
+});
+
+describe("ParseMultiLine function", function () {
+  const rowSplit = new RowSplit(new Converter());
+  const func = (lines: string[]): MultipleRowResult => {
+    return rowSplit.parseMultiLines(lines);
+  }
+  it("should convert lines to csv lines", function () {
+    var lines = [
+      "a,b,c,d",
+      "hello,world,csvtojson,abc",
+      "1,2,3,4"
+    ];
+    var res = func(lines);
+    assert.equal(res.rowsCells.length, 3);
+    assert.equal(res.partial, "");
+  });
+
+  it("should process line breaks", function () {
+    var lines = [
+      "a,b,c",
+      '15",hello,"ab',
+      "cde\"",
+      "\"b\"\"b\",cc,dd"
+    ];
+    var res = func(lines);
+    assert.equal(res.rowsCells.length, 3);
+    assert.equal(res.rowsCells[1][0], "15\"");
+    assert.equal(res.rowsCells[1][2], "ab\ncde");
+    assert.equal(res.rowsCells[2][0], "b\"b");
+    assert.equal(res.partial, "");
+  });
+
+  it("should return partial if line not closed", function () {
+    var lines = [
+      "a,b,c",
+      '15",hello,"ab',
+      "d,e,f"
+    ];
+    var res = func(lines);
+    assert.equal(res.rowsCells.length, 1);
+    assert.equal(res.partial, "15\",hello,\"ab\nd,e,f\n");
+  });
+});
+
+describe("RowSplit.parse function", function () {
+  const rowSplit = new RowSplit(new Converter());
+  const func = (str): RowSplitResult => {
+    return rowSplit.parse(str);
+  }
+  it("should split complete csv line", function () {
+    var str = "hello,world,csvtojson,awesome";
+    var res = func(str);
+    assert.equal(res.cells.length, 4);
+    assert.equal(res.closed, true);
+  });
+
+  it("should split incomplete csv line", function () {
+    var str = "hello,world,\"csvtojson,awesome";
+    var res = func(str);
+    assert.equal(res.closed, false);
+  });
+
+  it("should allow multiple line", function () {
+    var str = "\"he\"llo\",world,\"csvtojson,a\"\nwesome\"";
+    var res = func(str);
+    assert.equal(res.closed, true);
+    assert.equal(res.cells[2], 'csvtojson,a"\nwesome');
+  });
+  it("should allow blank quotes", () => {
+    const data = "a|^^|^b^";
+
+    const rowSplit = new RowSplit(new Converter({
+      delimiter: '|',
+      quote: '^',
+      noheader: true
+    }));
+    const res = rowSplit.parse(data);
+    assert.equal(res.cells[1], "");
+  })
+  it("should allow blank quotes in quotes", () => {
+    const data = 'a,"hello,this,"", test"';
+
+    const rowSplit = new RowSplit(new Converter({
+      noheader: true
+    }));
+    const res = rowSplit.parse(data);
+    assert.equal(res.cells[1], 'hello,this,", test');
+  })
+  it("should smart detect if an initial quote is only part of value ", () => {
+    const data = '"Weight" (kg),Error code,"Height" (m)';
+    const rowSplit = new RowSplit(new Converter({
+      noheader: true
+    }));
+    const res = rowSplit.parse(data);
+    assert.equal(res.cells.length, 3);
+    assert(res.closed);
+    assert.equal(res.cells[0],'"Weight" (kg)');
+    assert.equal(res.cells[1],'Error code');
+    assert.equal(res.cells[2],'"Height" (m)');
+    
+  })
+});

node_modules/csvtojson/src/rowSplit.ts

@@ -0,0 +1,235 @@
+import { CSVParseParam } from "./Parameters";
+import { Converter } from "./Converter";
+import { Fileline } from "./fileline";
+import getEol from "./getEol";
+import { filterArray, trimLeft, trimRight } from "./util";
+
+const defaulDelimiters = [",", "|", "\t", ";", ":"];
+export class RowSplit {
+  private quote: string;
+  private trim: boolean;
+  private escape: string;
+  private cachedRegExp: { [key: string]: RegExp } = {};
+  private delimiterEmitted = false;
+  private _needEmitDelimiter?: boolean = undefined;
+  private get needEmitDelimiter() {
+    if (this._needEmitDelimiter === undefined) {
+      this._needEmitDelimiter = this.conv.listeners("delimiter").length > 0;
+    }
+    return this._needEmitDelimiter;
+  }
+  constructor(private conv: Converter) {
+    this.quote = conv.parseParam.quote;
+    this.trim = conv.parseParam.trim;
+    this.escape = conv.parseParam.escape;
+  }
+  parse(fileline: Fileline): RowSplitResult {
+    if (fileline.length === 0 || (this.conv.parseParam.ignoreEmpty && fileline.trim().length === 0)) {
+      return { cells: [], closed: true };
+    }
+    const quote = this.quote;
+    const trim = this.trim;
+    const escape = this.escape;
+    if (this.conv.parseRuntime.delimiter instanceof Array || this.conv.parseRuntime.delimiter.toLowerCase() === "auto") {
+      this.conv.parseRuntime.delimiter = this.getDelimiter(fileline);
+
+    }
+    if (this.needEmitDelimiter && !this.delimiterEmitted) {
+      this.conv.emit("delimiter", this.conv.parseRuntime.delimiter);
+      this.delimiterEmitted = true;
+    }
+    const delimiter = this.conv.parseRuntime.delimiter;
+    const rowArr = fileline.split(delimiter);
+    if (quote === "off") {
+      if (trim) {
+        for (let i = 0; i < rowArr.length; i++) {
+          rowArr[i] = rowArr[i].trim();
+        }
+      }
+      return { cells: rowArr, closed: true };
+    } else {
+      return this.toCSVRow(rowArr, trim, quote, delimiter);
+    }
+
+  }
+  private toCSVRow(rowArr: string[], trim: boolean, quote: string, delimiter: string): RowSplitResult {
+    const row: string[] = [];
+    let inquote = false;
+    let quoteBuff = '';
+    for (let i = 0, rowLen = rowArr.length; i < rowLen; i++) {
+      let e = rowArr[i];
+      if (!inquote && trim) {
+        e = trimLeft(e);
+      }
+      const len = e.length;
+      if (!inquote) {
+        if (len === 2 && e === this.quote + this.quote) {
+          row.push("");
+          continue;
+        } else if (this.isQuoteOpen(e)) { //quote open
+          e = e.substr(1);
+          if (this.isQuoteClose(e)) { //quote close
+            e = e.substring(0, e.lastIndexOf(quote));
+            e = this.escapeQuote(e);
+            row.push(e);
+            continue;
+          } else if (e.indexOf(quote) !== -1) {
+            let count = 0;
+            let prev = "";
+            for (const c of e) {
+              // count quotes only if previous character is not escape char
+              if (c === quote && prev !== this.escape) {
+                count++;
+                prev = "";
+              } else {
+                // save previous char to temp variable
+                prev = c;
+              }
+            }
+            if (count % 2 === 1) {
+              if (trim) {
+                e = trimRight(e);
+              }
+              row.push(quote + e);
+              continue;
+            }else{
+              inquote = true;
+              quoteBuff += e;
+              continue;  
+            }
+          }
+          else {
+            inquote = true;
+            quoteBuff += e;
+            continue;
+          }
+        } else {
+          if (trim) {
+            e = trimRight(e);
+          }
+          row.push(e);
+          continue;
+        }
+      } else { //previous quote not closed
+        if (this.isQuoteClose(e)) { //close double quote
+          inquote = false;
+          e = e.substr(0, len - 1);
+          quoteBuff += delimiter + e;
+          quoteBuff = this.escapeQuote(quoteBuff);
+          if (trim) {
+            quoteBuff = trimRight(quoteBuff);
+          }
+          row.push(quoteBuff);
+          quoteBuff = "";
+        } else {
+          quoteBuff += delimiter + e;
+        }
+      }
+    }
+
+    // if (!inquote && param._needFilterRow) {
+    //   row = filterRow(row, param);
+    // }
+
+    return { cells: row, closed: !inquote };
+  }
+  private getDelimiter(fileline: Fileline): string {
+    let checker;
+    if (this.conv.parseParam.delimiter === "auto") {
+      checker = defaulDelimiters;
+    } else if (this.conv.parseParam.delimiter instanceof Array) {
+      checker = this.conv.parseParam.delimiter;
+    } else {
+      return this.conv.parseParam.delimiter;
+    }
+    let count = 0;
+    let rtn = ",";
+    checker.forEach(function (delim) {
+      const delimCount = fileline.split(delim).length;
+      if (delimCount > count) {
+        rtn = delim;
+        count = delimCount;
+      }
+    });
+    return rtn;
+  }
+  private isQuoteOpen(str: string): boolean {
+    const quote = this.quote;
+    const escape = this.escape;
+    return str[0] === quote && (
+      str[1] !== quote ||
+      str[1] === escape && (str[2] === quote || str.length === 2));
+  }
+  private isQuoteClose(str: string): boolean {
+    const quote = this.quote;
+    const escape = this.escape;
+    if (this.conv.parseParam.trim) {
+      str = trimRight(str);
+    }
+    let count = 0;
+    let idx = str.length - 1;
+    while (str[idx] === quote || str[idx] === escape) {
+      idx--;
+      count++;
+    }
+    return count % 2 !== 0;
+  }
+
+  // private twoDoubleQuote(str: string): string {
+  //   var twoQuote = this.quote + this.quote;
+  //   var curIndex = -1;
+  //   while ((curIndex = str.indexOf(twoQuote, curIndex)) > -1) {
+  //     str = str.substring(0, curIndex) + str.substring(++curIndex);
+  //   }
+  //   return str;
+  // }
+
+
+  private escapeQuote(segment: string): string {
+    const key = "es|" + this.quote + "|" + this.escape;
+    if (this.cachedRegExp[key] === undefined) {
+      this.cachedRegExp[key] = new RegExp('\\' + this.escape + '\\' + this.quote, 'g');
+    }
+    const regExp = this.cachedRegExp[key];
+    // console.log(regExp,segment);
+    return segment.replace(regExp, this.quote);
+  }
+  parseMultiLines(lines: Fileline[]): MultipleRowResult {
+    const csvLines: string[][] = [];
+    let left = "";
+    while (lines.length) {
+      const line = left + lines.shift();
+      const row = this.parse(line);
+      if (row.cells.length === 0 && this.conv.parseParam.ignoreEmpty) {
+        continue;
+      }
+      if (row.closed || this.conv.parseParam.alwaysSplitAtEOL) {
+        if (this.conv.parseRuntime.selectedColumns) {
+          csvLines.push(filterArray(row.cells, this.conv.parseRuntime.selectedColumns));
+        } else {
+          csvLines.push(row.cells);
+        }
+
+        left = "";
+      } else {
+        left = line + (getEol(line, this.conv.parseRuntime) || "\n");
+      }
+    }
+    return { rowsCells: csvLines, partial: left };
+  }
+}
+export interface MultipleRowResult {
+  rowsCells: string[][];
+  partial: string;
+}
+export interface RowSplitResult {
+  /**
+   * csv row array. ["a","b","c"]
+   */
+  cells: string[],
+  /**
+   * if the passed fileline is a complete row
+   */
+  closed: boolean
+}
+

node_modules/csvtojson/src/util.ts

@@ -0,0 +1,37 @@
+export function bufFromString(str: string): Buffer {
+  const length = Buffer.byteLength(str);
+  const buffer = Buffer.allocUnsafe
+    ? Buffer.allocUnsafe(length)
+    : new Buffer(length);
+  buffer.write(str);
+  return buffer;
+}
+
+export function emptyBuffer(): Buffer{
+  const buffer = Buffer.allocUnsafe
+    ? Buffer.allocUnsafe(0)
+    : new Buffer(0);
+  return buffer;
+}
+
+export function filterArray(arr: any[], filter: number[]): any[] {
+  const rtn: any[] = [];
+  for (let i = 0; i < arr.length; i++) {
+    if (filter.indexOf(i) > -1) {
+      rtn.push(arr[i]);
+    }
+  }
+  return rtn;
+}
+
+export const trimLeft=String.prototype.trimLeft?function trimLeftNative(str:string){
+  return str.trimLeft();
+}:function trimLeftRegExp(str:string){
+  return str.replace(/^\s+/, "");
+}
+
+export const trimRight=String.prototype.trimRight?function trimRightNative(str:string){
+  return str.trimRight();
+}:function trimRightRegExp(str:string){
+  return str.replace(/\s+$/, "");
+}

node_modules/csvtojson/src/worker.ts

@@ -0,0 +1,76 @@
+// import { Converter } from "./Converter";
+// import { Message, InitMessage, EOM } from "./ProcessFork";
+// import CSVError from "./CSVError";
+// import { CSVParseParam } from "./Parameters";
+// process.on("message", processMsg);
+// let conv: Converter;
+// function processMsg(msg: Message) {
+//   if (msg.cmd === "init") {
+//     const param = prepareParams((msg as InitMessage).params);
+//     param.fork = false;
+//     conv = new Converter(param);
+//     process.stdin.pipe(conv).pipe(process.stdout);
+//     conv.on("error", (err) => {
+//       if ((err as CSVError).line) {
+//         process.stderr.write(JSON.stringify({
+//           err: (err as CSVError).err,
+//           line: (err as CSVError).line,
+//           extra: (err as CSVError).extra
+//         }))
+//       } else {
+//         process.stderr.write(JSON.stringify({
+//           err: err.message,
+//           line: -1,
+//           extra: "Unknown error"
+//         }));
+//       }
+
+//     });
+//     conv.on("eol", (eol) => {
+//       // console.log("eol!!!",eol);
+//       if (process.send)
+//         process.send({ cmd: "eol", "value": eol });
+//     })
+//     conv.on("header", (header) => {
+//       if (process.send)
+//         process.send({ cmd: "header", "value": header });
+//     })
+//     conv.on("done", () => {
+//       const drained = process.stdout.write("", () => {
+//         if (drained) {
+//           gracelyExit();
+//         }
+//       });
+//       if (!drained) {
+//         process.stdout.on("drain", gracelyExit)
+//       }
+
+
+//       // process.stdout.write(EOM);
+//     })
+//     if (process.send) {
+//       process.send({ cmd: "inited" });
+//     }
+
+
+//   }
+// }
+// function gracelyExit(){
+//   setTimeout(()=>{
+//     conv.removeAllListeners();
+//     process.removeAllListeners();
+//   },50);
+// }
+// function prepareParams(p: any): CSVParseParam {
+//   if (p.ignoreColumns) {
+//     p.ignoreColumns = new RegExp(p.ignoreColumns.source, p.ignoreColumns.flags)
+//   }
+//   if (p.includeColumns) {
+//     p.includeColumns = new RegExp(p.includeColumns.source, p.includeColumns.flags)
+//   }
+//   return p;
+// }
+
+// process.on("disconnect", () => {
+//   process.exit(-1);
+// });