1/* @license 2Papa Parse 3v5.4.1 4https://github.com/mholt/PapaParse 5License: MIT 6*/ 7 8(function(root, factory) 9{ 10 /* globals define */ 11 if (typeof define === 'function' && define.amd) 12 { 13 // AMD. Register as an anonymous module. 14 define([], factory); 15 } 16 else if (typeof module === 'object' && typeof exports !== 'undefined') 17 { 18 // Node. Does not work with strict CommonJS, but 19 // only CommonJS-like environments that support module.exports, 20 // like Node. 21 module.exports = factory(); 22 } 23 else 24 { 25 // Browser globals (root is window) 26 root.Papa = factory(); 27 } 28 // in strict mode we cannot access arguments.callee, so we need a named reference to 29 // stringify the factory method for the blob worker 30 // eslint-disable-next-line func-name 31}(this, function moduleFactory() 32{ 33 'use strict'; 34 35 var global = (function() { 36 // alternative method, similar to `Function('return this')()` 37 // but without using `eval` (which is disabled when 38 // using Content Security Policy). 39 40 if (typeof self !== 'undefined') { return self; } 41 if (typeof window !== 'undefined') { return window; } 42 if (typeof global !== 'undefined') { return global; } 43 44 // When running tests none of the above have been defined 45 return {}; 46 })(); 47 48 49 function getWorkerBlob() { 50 var URL = global.URL || global.webkitURL || null; 51 var code = moduleFactory.toString(); 52 return Papa.BLOB_URL || (Papa.BLOB_URL = URL.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ", '(', code, ')();'], {type: 'text/javascript'}))); 53 } 54 55 var IS_WORKER = !global.document && !!global.postMessage, 56 IS_PAPA_WORKER = global.IS_PAPA_WORKER || false; 57 58 var workers = {}, workerIdCounter = 0; 59 60 var Papa = {}; 61 62 Papa.parse = CsvToJson; 63 Papa.unparse = JsonToCsv; 64 65 Papa.RECORD_SEP = String.fromCharCode(30); 66 Papa.UNIT_SEP = String.fromCharCode(31); 67 Papa.BYTE_ORDER_MARK = '\ufeff'; 68 Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK]; 69 Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker; 70 Papa.NODE_STREAM_INPUT = 1; 71 72 // Configurable chunk sizes for local and remote files, respectively 73 Papa.LocalChunkSize = 1024 * 1024 * 10; // 10 MB 74 Papa.RemoteChunkSize = 1024 * 1024 * 5; // 5 MB 75 Papa.DefaultDelimiter = ','; // Used if not specified and detection fails 76 77 // Exposed for testing and development only 78 Papa.Parser = Parser; 79 Papa.ParserHandle = ParserHandle; 80 Papa.NetworkStreamer = NetworkStreamer; 81 Papa.FileStreamer = FileStreamer; 82 Papa.StringStreamer = StringStreamer; 83 Papa.ReadableStreamStreamer = ReadableStreamStreamer; 84 if (typeof PAPA_BROWSER_CONTEXT === 'undefined') { 85 Papa.DuplexStreamStreamer = DuplexStreamStreamer; 86 } 87 88 if (global.jQuery) 89 { 90 var $ = global.jQuery; 91 $.fn.parse = function(options) 92 { 93 var config = options.config || {}; 94 var queue = []; 95 96 this.each(function(idx) 97 { 98 var supported = $(this).prop('tagName').toUpperCase() === 'INPUT' 99 && $(this).attr('type').toLowerCase() === 'file' 100 && global.FileReader; 101 102 if (!supported || !this.files || this.files.length === 0) 103 return true; // continue to next input element 104 105 for (var i = 0; i < this.files.length; i++) 106 { 107 queue.push({ 108 file: this.files[i], 109 inputElem: this, 110 instanceConfig: $.extend({}, config) 111 }); 112 } 113 }); 114 115 parseNextFile(); // begin parsing 116 return this; // maintains chainability 117 118 119 function parseNextFile() 120 { 121 if (queue.length === 0) 122 { 123 if (isFunction(options.complete)) 124 options.complete(); 125 return; 126 } 127 128 var f = queue[0]; 129 130 if (isFunction(options.before)) 131 { 132 var returned = options.before(f.file, f.inputElem); 133 134 if (typeof returned === 'object') 135 { 136 if (returned.action === 'abort') 137 { 138 error('AbortError', f.file, f.inputElem, returned.reason); 139 return; // Aborts all queued files immediately 140 } 141 else if (returned.action === 'skip') 142 { 143 fileComplete(); // parse the next file in the queue, if any 144 return; 145 } 146 else if (typeof returned.config === 'object') 147 f.instanceConfig = $.extend(f.instanceConfig, returned.config); 148 } 149 else if (returned === 'skip') 150 { 151 fileComplete(); // parse the next file in the queue, if any 152 return; 153 } 154 } 155 156 // Wrap up the user's complete callback, if any, so that ours also gets executed 157 var userCompleteFunc = f.instanceConfig.complete; 158 f.instanceConfig.complete = function(results) 159 { 160 if (isFunction(userCompleteFunc)) 161 userCompleteFunc(results, f.file, f.inputElem); 162 fileComplete(); 163 }; 164 165 Papa.parse(f.file, f.instanceConfig); 166 } 167 168 function error(name, file, elem, reason) 169 { 170 if (isFunction(options.error))
vendor: 17,540 bytes, lines 171-831
171 options.error({name: name}, file, elem, reason); 172 } 173 174 function fileComplete() 175 { 176 queue.splice(0, 1); 177 parseNextFile(); 178 } 179 }; 180 } 181 182 183 if (IS_PAPA_WORKER) 184 { 185 global.onmessage = workerThreadReceivedMessage; 186 } 187 188 189 190 191 function CsvToJson(_input, _config) 192 { 193 _config = _config || {}; 194 var dynamicTyping = _config.dynamicTyping || false; 195 if (isFunction(dynamicTyping)) { 196 _config.dynamicTypingFunction = dynamicTyping; 197 // Will be filled on first row call 198 dynamicTyping = {}; 199 } 200 _config.dynamicTyping = dynamicTyping; 201 202 _config.transform = isFunction(_config.transform) ? _config.transform : false; 203 204 if (_config.worker && Papa.WORKERS_SUPPORTED) 205 { 206 var w = newWorker(); 207 208 w.userStep = _config.step; 209 w.userChunk = _config.chunk; 210 w.userComplete = _config.complete; 211 w.userError = _config.error; 212 213 _config.step = isFunction(_config.step); 214 _config.chunk = isFunction(_config.chunk); 215 _config.complete = isFunction(_config.complete); 216 _config.error = isFunction(_config.error); 217 delete _config.worker; // prevent infinite loop 218 219 w.postMessage({ 220 input: _input, 221 config: _config, 222 workerId: w.id 223 }); 224 225 return; 226 } 227 228 var streamer = null; 229 if (_input === Papa.NODE_STREAM_INPUT && typeof PAPA_BROWSER_CONTEXT === 'undefined') 230 { 231 // create a node Duplex stream for use 232 // with .pipe 233 streamer = new DuplexStreamStreamer(_config); 234 return streamer.getStream(); 235 } 236 else if (typeof _input === 'string') 237 { 238 _input = stripBom(_input); 239 if (_config.download) 240 streamer = new NetworkStreamer(_config); 241 else 242 streamer = new StringStreamer(_config); 243 } 244 else if (_input.readable === true && isFunction(_input.read) && isFunction(_input.on)) 245 { 246 streamer = new ReadableStreamStreamer(_config); 247 } 248 else if ((global.File && _input instanceof File) || _input instanceof Object) // ...Safari. (see issue #106) 249 streamer = new FileStreamer(_config); 250 251 return streamer.stream(_input); 252 253 // Strip character from UTF-8 BOM encoded files that cause issue parsing the file 254 function stripBom(string) { 255 if (string.charCodeAt(0) === 0xfeff) { 256 return string.slice(1); 257 } 258 return string; 259 } 260 } 261 262 263 264 265 266 267 function JsonToCsv(_input, _config) 268 { 269 // Default configuration 270 271 /** whether to surround every datum with quotes */ 272 var _quotes = false; 273 274 /** whether to write headers */ 275 var _writeHeader = true; 276 277 /** delimiting character(s) */ 278 var _delimiter = ','; 279 280 /** newline character(s) */ 281 var _newline = '\r\n'; 282 283 /** quote character */ 284 var _quoteChar = '"'; 285 286 /** escaped quote character, either "" or <config.escapeChar>" */ 287 var _escapedQuote = _quoteChar + _quoteChar; 288 289 /** whether to skip empty lines */ 290 var _skipEmptyLines = false; 291 292 /** the columns (keys) we expect when we unparse objects */ 293 var _columns = null; 294 295 /** whether to prevent outputting cells that can be parsed as formulae by spreadsheet software (Excel and LibreOffice) */ 296 var _escapeFormulae = false; 297 298 unpackConfig(); 299 300 var quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), 'g'); 301 302 if (typeof _input === 'string') 303 _input = JSON.parse(_input); 304 305 if (Array.isArray(_input)) 306 { 307 if (!_input.length || Array.isArray(_input[0])) 308 return serialize(null, _input, _skipEmptyLines); 309 else if (typeof _input[0] === 'object') 310 return serialize(_columns || Object.keys(_input[0]), _input, _skipEmptyLines); 311 } 312 else if (typeof _input === 'object') 313 { 314 if (typeof _input.data === 'string') 315 _input.data = JSON.parse(_input.data); 316 317 if (Array.isArray(_input.data)) 318 { 319 if (!_input.fields) 320 _input.fields = _input.meta && _input.meta.fields || _columns; 321 322 if (!_input.fields) 323 _input.fields = Array.isArray(_input.data[0]) 324 ? _input.fields 325 : typeof _input.data[0] === 'object' 326 ? Object.keys(_input.data[0]) 327 : []; 328 329 if (!(Array.isArray(_input.data[0])) && typeof _input.data[0] !== 'object') 330 _input.data = [_input.data]; // handles input like [1,2,3] or ['asdf'] 331 } 332 333 return serialize(_input.fields || [], _input.data || [], _skipEmptyLines); 334 } 335 336 // Default (any valid paths should return before this) 337 throw new Error('Unable to serialize unrecognized input'); 338 339 340 function unpackConfig() 341 { 342 if (typeof _config !== 'object') 343 return; 344 345 if (typeof _config.delimiter === 'string' 346 && !Papa.BAD_DELIMITERS.filter(function(value) { return _config.delimiter.indexOf(value) !== -1; }).length) 347 { 348 _delimiter = _config.delimiter; 349 } 350 351 if (typeof _config.quotes === 'boolean' 352 || typeof _config.quotes === 'function' 353 || Array.isArray(_config.quotes)) 354 _quotes = _config.quotes; 355 356 if (typeof _config.skipEmptyLines === 'boolean' 357 || typeof _config.skipEmptyLines === 'string') 358 _skipEmptyLines = _config.skipEmptyLines; 359 360 if (typeof _config.newline === 'string') 361 _newline = _config.newline; 362 363 if (typeof _config.quoteChar === 'string') 364 _quoteChar = _config.quoteChar; 365 366 if (typeof _config.header === 'boolean') 367 _writeHeader = _config.header; 368 369 if (Array.isArray(_config.columns)) { 370 371 if (_config.columns.length === 0) throw new Error('Option columns is empty'); 372 373 _columns = _config.columns; 374 } 375 376 if (_config.escapeChar !== undefined) { 377 _escapedQuote = _config.escapeChar + _quoteChar; 378 } 379 380 if (typeof _config.escapeFormulae === 'boolean' || _config.escapeFormulae instanceof RegExp) { 381 _escapeFormulae = _config.escapeFormulae instanceof RegExp ? _config.escapeFormulae : /^[=+\-@\t\r].*$/; 382 } 383 } 384 385 /** The double for loop that iterates the data and writes out a CSV string including header row */ 386 function serialize(fields, data, skipEmptyLines) 387 { 388 var csv = ''; 389 390 if (typeof fields === 'string') 391 fields = JSON.parse(fields); 392 if (typeof data === 'string') 393 data = JSON.parse(data); 394 395 var hasHeader = Array.isArray(fields) && fields.length > 0; 396 var dataKeyedByField = !(Array.isArray(data[0])); 397 398 // If there a header row, write it first 399 if (hasHeader && _writeHeader) 400 { 401 for (var i = 0; i < fields.length; i++) 402 { 403 if (i > 0) 404 csv += _delimiter; 405 csv += safe(fields[i], i); 406 } 407 if (data.length > 0) 408 csv += _newline; 409 } 410 411 // Then write out the data 412 for (var row = 0; row < data.length; row++) 413 { 414 var maxCol = hasHeader ? fields.length : data[row].length; 415 416 var emptyLine = false; 417 var nullLine = hasHeader ? Object.keys(data[row]).length === 0 : data[row].length === 0; 418 if (skipEmptyLines && !hasHeader) 419 { 420 emptyLine = skipEmptyLines === 'greedy' ? data[row].join('').trim() === '' : data[row].length === 1 && data[row][0].length === 0; 421 } 422 if (skipEmptyLines === 'greedy' && hasHeader) { 423 var line = []; 424 for (var c = 0; c < maxCol; c++) { 425 var cx = dataKeyedByField ? fields[c] : c; 426 line.push(data[row][cx]); 427 } 428 emptyLine = line.join('').trim() === ''; 429 } 430 if (!emptyLine) 431 { 432 for (var col = 0; col < maxCol; col++) 433 { 434 if (col > 0 && !nullLine) 435 csv += _delimiter; 436 var colIdx = hasHeader && dataKeyedByField ? fields[col] : col; 437 csv += safe(data[row][colIdx], col); 438 } 439 if (row < data.length - 1 && (!skipEmptyLines || (maxCol > 0 && !nullLine))) 440 { 441 csv += _newline; 442 } 443 } 444 } 445 return csv; 446 } 447 448 /** Encloses a value around quotes if needed (makes a value safe for CSV insertion) */ 449 function safe(str, col) 450 { 451 if (typeof str === 'undefined' || str === null) 452 return ''; 453 454 if (str.constructor === Date) 455 return JSON.stringify(str).slice(1, 25); 456 457 var needsQuotes = false; 458 459 if (_escapeFormulae && typeof str === "string" && _escapeFormulae.test(str)) { 460 str = "'" + str; 461 needsQuotes = true; 462 } 463 464 var escapedQuoteStr = str.toString().replace(quoteCharRegex, _escapedQuote); 465 466 needsQuotes = needsQuotes 467 || _quotes === true 468 || (typeof _quotes === 'function' && _quotes(str, col)) 469 || (Array.isArray(_quotes) && _quotes[col]) 470 || hasAny(escapedQuoteStr, Papa.BAD_DELIMITERS) 471 || escapedQuoteStr.indexOf(_delimiter) > -1 472 || escapedQuoteStr.charAt(0) === ' ' 473 || escapedQuoteStr.charAt(escapedQuoteStr.length - 1) === ' '; 474 475 return needsQuotes ? _quoteChar + escapedQuoteStr + _quoteChar : escapedQuoteStr; 476 } 477 478 function hasAny(str, substrings) 479 { 480 for (var i = 0; i < substrings.length; i++) 481 if (str.indexOf(substrings[i]) > -1) 482 return true; 483 return false; 484 } 485 } 486 487 /** ChunkStreamer is the base prototype for various streamer implementations. */ 488 function ChunkStreamer(config) 489 { 490 this._handle = null; 491 this._finished = false; 492 this._completed = false; 493 this._halted = false; 494 this._input = null; 495 this._baseIndex = 0; 496 this._partialLine = ''; 497 this._rowCount = 0; 498 this._start = 0; 499 this._nextChunk = null; 500 this.isFirstChunk = true; 501 this._completeResults = { 502 data: [], 503 errors: [], 504 meta: {} 505 }; 506 replaceConfig.call(this, config); 507 508 this.parseChunk = function(chunk, isFakeChunk) 509 { 510 // First chunk pre-processing 511 if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) 512 { 513 var modifiedChunk = this._config.beforeFirstChunk(chunk); 514 if (modifiedChunk !== undefined) 515 chunk = modifiedChunk; 516 } 517 this.isFirstChunk = false; 518 this._halted = false; 519 520 // Rejoin the line we likely just split in two by chunking the file 521 var aggregate = this._partialLine + chunk; 522 this._partialLine = ''; 523 524 var results = this._handle.parse(aggregate, this._baseIndex, !this._finished); 525 526 if (this._handle.paused() || this._handle.aborted()) { 527 this._halted = true; 528 return; 529 } 530 531 var lastIndex = results.meta.cursor; 532 533 if (!this._finished) 534 { 535 this._partialLine = aggregate.substring(lastIndex - this._baseIndex); 536 this._baseIndex = lastIndex; 537 } 538 539 if (results && results.data) 540 this._rowCount += results.data.length; 541 542 var finishedIncludingPreview = this._finished || (this._config.preview && this._rowCount >= this._config.preview); 543 544 if (IS_PAPA_WORKER) 545 { 546 global.postMessage({ 547 results: results, 548 workerId: Papa.WORKER_ID, 549 finished: finishedIncludingPreview 550 }); 551 } 552 else if (isFunction(this._config.chunk) && !isFakeChunk) 553 { 554 this._config.chunk(results, this._handle); 555 if (this._handle.paused() || this._handle.aborted()) { 556 this._halted = true; 557 return; 558 } 559 results = undefined; 560 this._completeResults = undefined; 561 } 562 563 if (!this._config.step && !this._config.chunk) { 564 this._completeResults.data = this._completeResults.data.concat(results.data); 565 this._completeResults.errors = this._completeResults.errors.concat(results.errors); 566 this._completeResults.meta = results.meta; 567 } 568 569 if (!this._completed && finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) { 570 this._config.complete(this._completeResults, this._input); 571 this._completed = true; 572 } 573 574 if (!finishedIncludingPreview && (!results || !results.meta.paused)) 575 this._nextChunk(); 576 577 return results; 578 }; 579 580 this._sendError = function(error) 581 { 582 if (isFunction(this._config.error)) 583 this._config.error(error); 584 else if (IS_PAPA_WORKER && this._config.error) 585 { 586 global.postMessage({ 587 workerId: Papa.WORKER_ID, 588 error: error, 589 finished: false 590 }); 591 } 592 }; 593 594 function replaceConfig(config) 595 { 596 // Deep-copy the config so we can edit it 597 var configCopy = copy(config); 598 configCopy.chunkSize = parseInt(configCopy.chunkSize); // parseInt VERY important so we don't concatenate strings! 599 if (!config.step && !config.chunk) 600 configCopy.chunkSize = null; // disable Range header if not streaming; bad values break IIS - see issue #196 601 this._handle = new ParserHandle(configCopy); 602 this._handle.streamer = this; 603 this._config = configCopy; // persist the copy to the caller 604 } 605 } 606 607 608 function NetworkStreamer(config) 609 { 610 config = config || {}; 611 if (!config.chunkSize) 612 config.chunkSize = Papa.RemoteChunkSize; 613 ChunkStreamer.call(this, config); 614 615 var xhr; 616 617 if (IS_WORKER) 618 { 619 this._nextChunk = function() 620 { 621 this._readChunk(); 622 this._chunkLoaded(); 623 }; 624 } 625 else 626 { 627 this._nextChunk = function() 628 { 629 this._readChunk(); 630 }; 631 } 632 633 this.stream = function(url) 634 { 635 this._input = url; 636 this._nextChunk(); // Starts streaming 637 }; 638 639 this._readChunk = function() 640 { 641 if (this._finished) 642 { 643 this._chunkLoaded(); 644 return; 645 } 646 647 xhr = new XMLHttpRequest(); 648 649 if (this._config.withCredentials) 650 { 651 xhr.withCredentials = this._config.withCredentials; 652 } 653 654 if (!IS_WORKER) 655 { 656 xhr.onload = bindFunction(this._chunkLoaded, this); 657 xhr.onerror = bindFunction(this._chunkError, this); 658 } 659 660 xhr.open(this._config.downloadRequestBody ? 'POST' : 'GET', this._input, !IS_WORKER); 661 // Headers can only be set when once the request state is OPENED 662 if (this._config.downloadRequestHeaders) 663 { 664 var headers = this._config.downloadRequestHeaders; 665 666 for (var headerName in headers) 667 { 668 xhr.setRequestHeader(headerName, headers[headerName]); 669 } 670 } 671 672 if (this._config.chunkSize) 673 { 674 var end = this._start + this._config.chunkSize - 1; // minus one because byte range is inclusive 675 xhr.setRequestHeader('Range', 'bytes=' + this._start + '-' + end); 676 } 677 678 try { 679 xhr.send(this._config.downloadRequestBody); 680 } 681 catch (err) { 682 this._chunkError(err.message); 683 } 684 685 if (IS_WORKER && xhr.status === 0) 686 this._chunkError(); 687 }; 688 689 this._chunkLoaded = function() 690 { 691 if (xhr.readyState !== 4) 692 return; 693 694 if (xhr.status < 200 || xhr.status >= 400) 695 { 696 this._chunkError(); 697 return; 698 } 699 700 // Use chunckSize as it may be a diference on reponse lentgh due to characters with more than 1 byte 701 this._start += this._config.chunkSize ? this._config.chunkSize : xhr.responseText.length; 702 this._finished = !this._config.chunkSize || this._start >= getFileSize(xhr); 703 this.parseChunk(xhr.responseText); 704 }; 705 706 this._chunkError = function(errorMessage) 707 { 708 var errorText = xhr.statusText || errorMessage; 709 this._sendError(new Error(errorText)); 710 }; 711 712 function getFileSize(xhr) 713 { 714 var contentRange = xhr.getResponseHeader('Content-Range'); 715 if (contentRange === null) { // no content range, then finish! 716 return -1; 717 } 718 return parseInt(contentRange.substring(contentRange.lastIndexOf('/') + 1)); 719 } 720 } 721 NetworkStreamer.prototype = Object.create(ChunkStreamer.prototype); 722 NetworkStreamer.prototype.constructor = NetworkStreamer; 723 724 725 function FileStreamer(config) 726 { 727 config = config || {}; 728 if (!config.chunkSize) 729 config.chunkSize = Papa.LocalChunkSize; 730 ChunkStreamer.call(this, config); 731 732 var reader, slice; 733 734 // FileReader is better than FileReaderSync (even in worker) - see http://stackoverflow.com/q/24708649/1048862 735 // But Firefox is a pill, too - see issue #76: https://github.com/mholt/PapaParse/issues/76 736 var usingAsyncReader = typeof FileReader !== 'undefined'; // Safari doesn't consider it a function - see issue #105 737 738 this.stream = function(file) 739 { 740 this._input = file; 741 slice = file.slice || file.webkitSlice || file.mozSlice; 742 743 if (usingAsyncReader) 744 { 745 reader = new FileReader(); // Preferred method of reading files, even in workers 746 reader.onload = bindFunction(this._chunkLoaded, this); 747 reader.onerror = bindFunction(this._chunkError, this); 748 } 749 else 750 reader = new FileReaderSync(); // Hack for running in a web worker in Firefox 751 752 this._nextChunk(); // Starts streaming 753 }; 754 755 this._nextChunk = function() 756 { 757 if (!this._finished && (!this._config.preview || this._rowCount < this._config.preview)) 758 this._readChunk(); 759 }; 760 761 this._readChunk = function() 762 { 763 var input = this._input; 764 if (this._config.chunkSize) 765 { 766 var end = Math.min(this._start + this._config.chunkSize, this._input.size); 767 input = slice.call(input, this._start, end); 768 } 769 var txt = reader.readAsText(input, this._config.encoding); 770 if (!usingAsyncReader) 771 this._chunkLoaded({ target: { result: txt } }); // mimic the async signature 772 }; 773 774 this._chunkLoaded = function(event) 775 { 776 // Very important to increment start each time before handling results 777 this._start += this._config.chunkSize; 778 this._finished = !this._config.chunkSize || this._start >= this._input.size; 779 this.parseChunk(event.target.result); 780 }; 781 782 this._chunkError = function() 783 { 784 this._sendError(reader.error); 785 }; 786 787 } 788 FileStreamer.prototype = Object.create(ChunkStreamer.prototype); 789 FileStreamer.prototype.constructor = FileStreamer; 790 791 792 function StringStreamer(config) 793 { 794 config = config || {}; 795 ChunkStreamer.call(this, config); 796 797 var remaining; 798 this.stream = function(s) 799 { 800 remaining = s; 801 return this._nextChunk(); 802 }; 803 this._nextChunk = function() 804 { 805 if (this._finished) return; 806 var size = this._config.chunkSize; 807 var chunk; 808 if(size) { 809 chunk = remaining.substring(0, size); 810 remaining = remaining.substring(size); 811 } else { 812 chunk = remaining; 813 remaining = ''; 814 } 815 this._finished = !remaining; 816 return this.parseChunk(chunk); 817 }; 818 } 819 StringStreamer.prototype = Object.create(StringStreamer.prototype); 820 StringStreamer.prototype.constructor = StringStreamer; 821 822 823 function ReadableStreamStreamer(config) 824 { 825 config = config || {}; 826 827 ChunkStreamer.call(this, config); 828 829 var queue = []; 830 var parseOnData = true; 831 var streamHasEnded = false;
vendor: 6,623 bytes, lines 832-1080
832 833 this.pause = function() 834 { 835 ChunkStreamer.prototype.pause.apply(this, arguments); 836 this._input.pause(); 837 }; 838 839 this.resume = function() 840 { 841 ChunkStreamer.prototype.resume.apply(this, arguments); 842 this._input.resume(); 843 }; 844 845 this.stream = function(stream) 846 { 847 this._input = stream; 848 849 this._input.on('data', this._streamData); 850 this._input.on('end', this._streamEnd); 851 this._input.on('error', this._streamError); 852 }; 853 854 this._checkIsFinished = function() 855 { 856 if (streamHasEnded && queue.length === 1) { 857 this._finished = true; 858 } 859 }; 860 861 this._nextChunk = function() 862 { 863 this._checkIsFinished(); 864 if (queue.length) 865 { 866 this.parseChunk(queue.shift()); 867 } 868 else 869 { 870 parseOnData = true; 871 } 872 }; 873 874 this._streamData = bindFunction(function(chunk) 875 { 876 try 877 { 878 queue.push(typeof chunk === 'string' ? chunk : chunk.toString(this._config.encoding)); 879 880 if (parseOnData) 881 { 882 parseOnData = false; 883 this._checkIsFinished(); 884 this.parseChunk(queue.shift()); 885 } 886 } 887 catch (error) 888 { 889 this._streamError(error); 890 } 891 }, this); 892 893 this._streamError = bindFunction(function(error) 894 { 895 this._streamCleanUp(); 896 this._sendError(error); 897 }, this); 898 899 this._streamEnd = bindFunction(function() 900 { 901 this._streamCleanUp(); 902 streamHasEnded = true; 903 this._streamData(''); 904 }, this); 905 906 this._streamCleanUp = bindFunction(function() 907 { 908 this._input.removeListener('data', this._streamData); 909 this._input.removeListener('end', this._streamEnd); 910 this._input.removeListener('error', this._streamError); 911 }, this); 912 } 913 ReadableStreamStreamer.prototype = Object.create(ChunkStreamer.prototype); 914 ReadableStreamStreamer.prototype.constructor = ReadableStreamStreamer; 915 916 917 function DuplexStreamStreamer(_config) { 918 var Duplex = require('stream').Duplex; 919 var config = copy(_config); 920 var parseOnWrite = true; 921 var writeStreamHasFinished = false; 922 var parseCallbackQueue = []; 923 var stream = null; 924 925 this._onCsvData = function(results) 926 { 927 var data = results.data; 928 if (!stream.push(data) && !this._handle.paused()) { 929 // the writeable consumer buffer has filled up 930 // so we need to pause until more items 931 // can be processed 932 this._handle.pause(); 933 } 934 }; 935 936 this._onCsvComplete = function() 937 { 938 // node will finish the read stream when 939 // null is pushed 940 stream.push(null); 941 }; 942 943 config.step = bindFunction(this._onCsvData, this); 944 config.complete = bindFunction(this._onCsvComplete, this); 945 ChunkStreamer.call(this, config); 946 947 this._nextChunk = function() 948 { 949 if (writeStreamHasFinished && parseCallbackQueue.length === 1) { 950 this._finished = true; 951 } 952 if (parseCallbackQueue.length) { 953 parseCallbackQueue.shift()(); 954 } else { 955 parseOnWrite = true; 956 } 957 }; 958 959 this._addToParseQueue = function(chunk, callback) 960 { 961 // add to queue so that we can indicate 962 // completion via callback 963 // node will automatically pause the incoming stream 964 // when too many items have been added without their 965 // callback being invoked 966 parseCallbackQueue.push(bindFunction(function() { 967 this.parseChunk(typeof chunk === 'string' ? chunk : chunk.toString(config.encoding)); 968 if (isFunction(callback)) { 969 return callback(); 970 } 971 }, this)); 972 if (parseOnWrite) { 973 parseOnWrite = false; 974 this._nextChunk(); 975 } 976 }; 977 978 this._onRead = function() 979 { 980 if (this._handle.paused()) { 981 // the writeable consumer can handle more data 982 // so resume the chunk parsing 983 this._handle.resume(); 984 } 985 }; 986 987 this._onWrite = function(chunk, encoding, callback) 988 { 989 this._addToParseQueue(chunk, callback); 990 }; 991 992 this._onWriteComplete = function() 993 { 994 writeStreamHasFinished = true; 995 // have to write empty string 996 // so parser knows its done 997 this._addToParseQueue(''); 998 }; 999 1000 this.getStream = function() 1001 { 1002 return stream; 1003 }; 1004 stream = new Duplex({ 1005 readableObjectMode: true, 1006 decodeStrings: false, 1007 read: bindFunction(this._onRead, this), 1008 write: bindFunction(this._onWrite, this) 1009 }); 1010 stream.once('finish', bindFunction(this._onWriteComplete, this)); 1011 } 1012 if (typeof PAPA_BROWSER_CONTEXT === 'undefined') { 1013 DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype); 1014 DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer; 1015 } 1016 1017 1018 // Use one ParserHandle per entire CSV file or string 1019 function ParserHandle(_config) 1020 { 1021 // One goal is to minimize the use of regular expressions... 1022 var MAX_FLOAT = Math.pow(2, 53); 1023 var MIN_FLOAT = -MAX_FLOAT; 1024 var FLOAT = /^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/; 1025 var ISO_DATE = /^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/; 1026 var self = this; 1027 var _stepCounter = 0; // Number of times step was called (number of rows parsed) 1028 var _rowCounter = 0; // Number of rows that have been parsed so far 1029 var _input; // The input being parsed 1030 var _parser; // The core parser being used 1031 var _paused = false; // Whether we are paused or not 1032 var _aborted = false; // Whether the parser has aborted or not 1033 var _delimiterError; // Temporary state between delimiter detection and processing results 1034 var _fields = []; // Fields are from the header row of the input, if there is one 1035 var _results = { // The last results returned from the parser 1036 data: [], 1037 errors: [], 1038 meta: {} 1039 }; 1040 1041 if (isFunction(_config.step)) 1042 { 1043 var userStep = _config.step; 1044 _config.step = function(results) 1045 { 1046 _results = results; 1047 1048 if (needsHeaderRow()) 1049 processResults(); 1050 else // only call user's step function after header row 1051 { 1052 processResults(); 1053 1054 // It's possbile that this line was empty and there's no row here after all 1055 if (_results.data.length === 0) 1056 return; 1057 1058 _stepCounter += results.data.length; 1059 if (_config.preview && _stepCounter > _config.preview) 1060 _parser.abort(); 1061 else { 1062 _results.data = _results.data[0]; 1063 userStep(_results, self); 1064 } 1065 } 1066 }; 1067 } 1068 1069 /** 1070 * Parses input. Most users won't need, and shouldn't mess with, the baseIndex 1071 * and ignoreLastRow parameters. They are used by streamers (wrapper functions) 1072 * when an input comes in multiple chunks, like from a file. 1073 */ 1074 this.parse = function(input, baseIndex, ignoreLastRow) 1075 { 1076 var quoteChar = _config.quoteChar || '"'; 1077 if (!_config.newline) 1078 _config.newline = guessLineEndings(input, quoteChar); 1079 1080 _delimiterError = false;
vendor: 7,935 bytes, lines 1081-1383
1081 if (!_config.delimiter) 1082 { 1083 var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments, _config.delimitersToGuess); 1084 if (delimGuess.successful) 1085 _config.delimiter = delimGuess.bestDelimiter; 1086 else 1087 { 1088 _delimiterError = true; // add error after parsing (otherwise it would be overwritten) 1089 _config.delimiter = Papa.DefaultDelimiter; 1090 } 1091 _results.meta.delimiter = _config.delimiter; 1092 } 1093 else if(isFunction(_config.delimiter)) 1094 { 1095 _config.delimiter = _config.delimiter(input); 1096 _results.meta.delimiter = _config.delimiter; 1097 } 1098 1099 var parserConfig = copy(_config); 1100 if (_config.preview && _config.header) 1101 parserConfig.preview++; // to compensate for header row 1102 1103 _input = input; 1104 _parser = new Parser(parserConfig); 1105 _results = _parser.parse(_input, baseIndex, ignoreLastRow); 1106 processResults(); 1107 return _paused ? { meta: { paused: true } } : (_results || { meta: { paused: false } }); 1108 }; 1109 1110 this.paused = function() 1111 { 1112 return _paused; 1113 }; 1114 1115 this.pause = function() 1116 { 1117 _paused = true; 1118 _parser.abort(); 1119 1120 // If it is streaming via "chunking", the reader will start appending correctly already so no need to substring, 1121 // otherwise we can get duplicate content within a row 1122 _input = isFunction(_config.chunk) ? "" : _input.substring(_parser.getCharIndex()); 1123 }; 1124 1125 this.resume = function() 1126 { 1127 if(self.streamer._halted) { 1128 _paused = false; 1129 self.streamer.parseChunk(_input, true); 1130 } else { 1131 // Bugfix: #636 In case the processing hasn't halted yet 1132 // wait for it to halt in order to resume 1133 setTimeout(self.resume, 3); 1134 } 1135 }; 1136 1137 this.aborted = function() 1138 { 1139 return _aborted; 1140 }; 1141 1142 this.abort = function() 1143 { 1144 _aborted = true; 1145 _parser.abort(); 1146 _results.meta.aborted = true; 1147 if (isFunction(_config.complete)) 1148 _config.complete(_results); 1149 _input = ''; 1150 }; 1151 1152 function testEmptyLine(s) { 1153 return _config.skipEmptyLines === 'greedy' ? s.join('').trim() === '' : s.length === 1 && s[0].length === 0; 1154 } 1155 1156 function testFloat(s) { 1157 if (FLOAT.test(s)) { 1158 var floatValue = parseFloat(s); 1159 if (floatValue > MIN_FLOAT && floatValue < MAX_FLOAT) { 1160 return true; 1161 } 1162 } 1163 return false; 1164 } 1165 1166 function processResults() 1167 { 1168 if (_results && _delimiterError) 1169 { 1170 addError('Delimiter', 'UndetectableDelimiter', 'Unable to auto-detect delimiting character; defaulted to \'' + Papa.DefaultDelimiter + '\''); 1171 _delimiterError = false; 1172 } 1173 1174 if (_config.skipEmptyLines) 1175 { 1176 _results.data = _results.data.filter(function(d) { 1177 return !testEmptyLine(d); 1178 }); 1179 } 1180 1181 if (needsHeaderRow()) 1182 fillHeaderFields(); 1183 1184 return applyHeaderAndDynamicTypingAndTransformation(); 1185 } 1186 1187 function needsHeaderRow() 1188 { 1189 return _config.header && _fields.length === 0; 1190 } 1191 1192 function fillHeaderFields() 1193 { 1194 if (!_results) 1195 return; 1196 1197 function addHeader(header, i) 1198 { 1199 if (isFunction(_config.transformHeader)) 1200 header = _config.transformHeader(header, i); 1201 1202 _fields.push(header); 1203 } 1204 1205 if (Array.isArray(_results.data[0])) 1206 { 1207 for (var i = 0; needsHeaderRow() && i < _results.data.length; i++) 1208 _results.data[i].forEach(addHeader); 1209 1210 _results.data.splice(0, 1); 1211 } 1212 // if _results.data[0] is not an array, we are in a step where _results.data is the row. 1213 else 1214 _results.data.forEach(addHeader); 1215 } 1216 1217 function shouldApplyDynamicTyping(field) { 1218 // Cache function values to avoid calling it for each row 1219 if (_config.dynamicTypingFunction && _config.dynamicTyping[field] === undefined) { 1220 _config.dynamicTyping[field] = _config.dynamicTypingFunction(field); 1221 } 1222 return (_config.dynamicTyping[field] || _config.dynamicTyping) === true; 1223 } 1224 1225 function parseDynamic(field, value) 1226 { 1227 if (shouldApplyDynamicTyping(field)) 1228 { 1229 if (value === 'true' || value === 'TRUE') 1230 return true; 1231 else if (value === 'false' || value === 'FALSE') 1232 return false; 1233 else if (testFloat(value)) 1234 return parseFloat(value); 1235 else if (ISO_DATE.test(value)) 1236 return new Date(value); 1237 else 1238 return (value === '' ? null : value); 1239 } 1240 return value; 1241 } 1242 1243 function applyHeaderAndDynamicTypingAndTransformation() 1244 { 1245 if (!_results || (!_config.header && !_config.dynamicTyping && !_config.transform)) 1246 return _results; 1247 1248 function processRow(rowSource, i) 1249 { 1250 var row = _config.header ? {} : []; 1251 1252 var j; 1253 for (j = 0; j < rowSource.length; j++) 1254 { 1255 var field = j; 1256 var value = rowSource[j]; 1257 1258 if (_config.header) 1259 field = j >= _fields.length ? '__parsed_extra' : _fields[j]; 1260 1261 if (_config.transform) 1262 value = _config.transform(value,field); 1263 1264 value = parseDynamic(field, value); 1265 1266 if (field === '__parsed_extra') 1267 { 1268 row[field] = row[field] || []; 1269 row[field].push(value); 1270 } 1271 else 1272 row[field] = value; 1273 } 1274 1275 1276 if (_config.header) 1277 { 1278 if (j > _fields.length) 1279 addError('FieldMismatch', 'TooManyFields', 'Too many fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i); 1280 else if (j < _fields.length) 1281 addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i); 1282 } 1283 1284 return row; 1285 } 1286 1287 var incrementBy = 1; 1288 if (!_results.data.length || Array.isArray(_results.data[0])) 1289 { 1290 _results.data = _results.data.map(processRow); 1291 incrementBy = _results.data.length; 1292 } 1293 else 1294 _results.data = processRow(_results.data, 0); 1295 1296 1297 if (_config.header && _results.meta) 1298 _results.meta.fields = _fields; 1299 1300 _rowCounter += incrementBy; 1301 return _results; 1302 } 1303 1304 function guessDelimiter(input, newline, skipEmptyLines, comments, delimitersToGuess) { 1305 var bestDelim, bestDelta, fieldCountPrevRow, maxFieldCount; 1306 1307 delimitersToGuess = delimitersToGuess || [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP]; 1308 1309 for (var i = 0; i < delimitersToGuess.length; i++) { 1310 var delim = delimitersToGuess[i]; 1311 var delta = 0, avgFieldCount = 0, emptyLinesCount = 0; 1312 fieldCountPrevRow = undefined; 1313 1314 var preview = new Parser({ 1315 comments: comments, 1316 delimiter: delim, 1317 newline: newline, 1318 preview: 10 1319 }).parse(input); 1320 1321 for (var j = 0; j < preview.data.length; j++) { 1322 if (skipEmptyLines && testEmptyLine(preview.data[j])) { 1323 emptyLinesCount++; 1324 continue; 1325 } 1326 var fieldCount = preview.data[j].length; 1327 avgFieldCount += fieldCount; 1328 1329 if (typeof fieldCountPrevRow === 'undefined') { 1330 fieldCountPrevRow = fieldCount; 1331 continue; 1332 } 1333 else if (fieldCount > 0) { 1334 delta += Math.abs(fieldCount - fieldCountPrevRow); 1335 fieldCountPrevRow = fieldCount; 1336 } 1337 } 1338 1339 if (preview.data.length > 0) 1340 avgFieldCount /= (preview.data.length - emptyLinesCount); 1341 1342 if ((typeof bestDelta === 'undefined' || delta <= bestDelta) 1343 && (typeof maxFieldCount === 'undefined' || avgFieldCount > maxFieldCount) && avgFieldCount > 1.99) { 1344 bestDelta = delta; 1345 bestDelim = delim; 1346 maxFieldCount = avgFieldCount; 1347 } 1348 } 1349 1350 _config.delimiter = bestDelim; 1351 1352 return { 1353 successful: !!bestDelim, 1354 bestDelimiter: bestDelim 1355 }; 1356 } 1357 1358 function guessLineEndings(input, quoteChar) 1359 { 1360 input = input.substring(0, 1024 * 1024); // max length 1 MB 1361 // Replace all the text inside quotes 1362 var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm'); 1363 input = input.replace(re, ''); 1364 1365 var r = input.split('\r'); 1366 1367 var n = input.split('\n'); 1368 1369 var nAppearsFirst = (n.length > 1 && n[0].length < r[0].length); 1370 1371 if (r.length === 1 || nAppearsFirst) 1372 return '\n'; 1373 1374 var numWithN = 0; 1375 for (var i = 0; i < r.length; i++) 1376 { 1377 if (r[i][0] === '\n') 1378 numWithN++; 1379 } 1380 1381 return numWithN >= r.length / 2 ? '\r\n' : '\r'; 1382 } 1383
1384 function addError(type, code, msg, row) 1385 { 1386 var error = { 1387 type: type, 1388 code: code, 1389 message: msg 1390 }; 1391 if(row !== undefined) { 1392 error.row = row; 1393 } 1394 _results.errors.push(error); 1395 } 1396 } 1397 1398 /** https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions */ 1399 function escapeRegExp(string) 1400 { 1401 return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string 1402 } 1403 1404 /** The core parser implements speedy and correct CSV parsing */ 1405 function Parser(config) 1406 { 1407 // Unpack the config object 1408 config = config || {}; 1409 var delim = config.delimiter; 1410 var newline = config.newline; 1411 var comments = config.comments; 1412 var step = config.step; 1413 var preview = config.preview; 1414 var fastMode = config.fastMode; 1415 var quoteChar; 1416 if (config.quoteChar === undefined || config.quoteChar === null) { 1417 quoteChar = '"'; 1418 } else { 1419 quoteChar = config.quoteChar; 1420 } 1421 var escapeChar = quoteChar; 1422 if (config.escapeChar !== undefined) { 1423 escapeChar = config.escapeChar; 1424 } 1425 1426 // Delimiter must be valid 1427 if (typeof delim !== 'string' 1428 || Papa.BAD_DELIMITERS.indexOf(delim) > -1) 1429 delim = ','; 1430 1431 // Comment character must be valid 1432 if (comments === delim) 1433 throw new Error('Comment character same as delimiter'); 1434 else if (comments === true) 1435 comments = '#'; 1436 else if (typeof comments !== 'string' 1437 || Papa.BAD_DELIMITERS.indexOf(comments) > -1) 1438 comments = false; 1439 1440 // Newline must be valid: \r, \n, or \r\n 1441 if (newline !== '\n' && newline !== '\r' && newline !== '\r\n') 1442 newline = '\n'; 1443 1444 // We're gonna need these at the Parser scope 1445 var cursor = 0; 1446 var aborted = false; 1447 1448 this.parse = function(input, baseIndex, ignoreLastRow) 1449 { 1450 // For some reason, in Chrome, this speeds things up (!?) 1451 if (typeof input !== 'string') 1452 throw new Error('Input must be a string'); 1453 1454 // We don't need to compute some of these every time parse() is called, 1455 // but having them in a more local scope seems to perform better 1456 var inputLen = input.length, 1457 delimLen = delim.length, 1458 newlineLen = newline.length, 1459 commentsLen = comments.length; 1460 var stepIsFunction = isFunction(step); 1461 1462 // Establish starting state 1463 cursor = 0; 1464 var data = [], errors = [], row = [], lastCursor = 0; 1465 1466 if (!input) 1467 return returnable(); 1468 1469 // Rename headers if there are duplicates 1470 if (config.header && !baseIndex) 1471 { 1472 var firstLine = input.split(newline)[0]; 1473 var headers = firstLine.split(delim); 1474 var separator = '_'; 1475 var headerMap = []; 1476 var headerCount = {}; 1477 var duplicateHeaders = false; 1478 1479 for (var j in headers) { 1480 var header = headers[j]; 1481 if (isFunction(config.transformHeader)) 1482 header = config.transformHeader(header, j); 1483 var headerName = header; 1484 1485 var count = headerCount[header] || 0; 1486 if (count > 0) { 1487 duplicateHeaders = true; 1488 headerName = header + separator + count; 1489 } 1490 headerCount[header] = count + 1; 1491 // In case it already exists, we add more separtors 1492 while (headerMap.includes(headerName)) { 1493 headerName = headerName + separator + count; 1494 } 1495 headerMap.push(headerName); 1496 } 1497 if (duplicateHeaders) { 1498 var editedInput = input.split(newline); 1499 editedInput[0] = headerMap.join(delim); 1500 input = editedInput.join(newline); 1501 } 1502 } 1503 if (fastMode || (fastMode !== false && input.indexOf(quoteChar) === -1)) 1504 { 1505 var rows = input.split(newline); 1506 for (var i = 0; i < rows.length; i++) 1507 { 1508 row = rows[i]; 1509 cursor += row.length; 1510 if (i !== rows.length - 1) 1511 cursor += newline.length; 1512 else if (ignoreLastRow) 1513 return returnable(); 1514 if (comments && row.substring(0, commentsLen) === comments) 1515 continue; 1516 if (stepIsFunction) 1517 { 1518 data = []; 1519 pushRow(row.split(delim)); 1520 doStep(); 1521 if (aborted) 1522 return returnable(); 1523 } 1524 else 1525 pushRow(row.split(delim)); 1526 if (preview && i >= preview) 1527 { 1528 data = data.slice(0, preview); 1529 return returnable(true); 1530 } 1531 } 1532 return returnable(); 1533 } 1534 1535 var nextDelim = input.indexOf(delim, cursor); 1536 var nextNewline = input.indexOf(newline, cursor); 1537 var quoteCharRegex = new RegExp(escapeRegExp(escapeChar) + escapeRegExp(quoteChar), 'g'); 1538 var quoteSearch = input.indexOf(quoteChar, cursor); 1539 1540 // Parser loop 1541 for (;;) 1542 { 1543 // Field has opening quote 1544 if (input[cursor] === quoteChar) 1545 { 1546 // Start our search for the closing quote where the cursor is 1547 quoteSearch = cursor; 1548 1549 // Skip the opening quote 1550 cursor++; 1551 1552 for (;;) 1553 { 1554 // Find closing quote 1555 quoteSearch = input.indexOf(quoteChar, quoteSearch + 1); 1556 1557 //No other quotes are found - no other delimiters 1558 if (quoteSearch === -1) 1559 { 1560 if (!ignoreLastRow) { 1561 // No closing quote... what a pity 1562 errors.push({ 1563 type: 'Quotes', 1564 code: 'MissingQuotes', 1565 message: 'Quoted field unterminated', 1566 row: data.length, // row has yet to be inserted 1567 index: cursor 1568 }); 1569 } 1570 return finish(); 1571 } 1572 1573 // Closing quote at EOF 1574 if (quoteSearch === inputLen - 1) 1575 { 1576 var value = input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar); 1577 return finish(value); 1578 } 1579 1580 // If this quote is escaped, it's part of the data; skip it 1581 // If the quote character is the escape character, then check if the next character is the escape character 1582 if (quoteChar === escapeChar && input[quoteSearch + 1] === escapeChar) 1583 { 1584 quoteSearch++; 1585 continue; 1586 } 1587 1588 // If the quote character is not the escape character, then check if the previous character was the escape character 1589 if (quoteChar !== escapeChar && quoteSearch !== 0 && input[quoteSearch - 1] === escapeChar) 1590 { 1591 continue; 1592 } 1593 1594 if(nextDelim !== -1 && nextDelim < (quoteSearch + 1)) { 1595 nextDelim = input.indexOf(delim, (quoteSearch + 1)); 1596 } 1597 if(nextNewline !== -1 && nextNewline < (quoteSearch + 1)) { 1598 nextNewline = input.indexOf(newline, (quoteSearch + 1)); 1599 } 1600 // Check up to nextDelim or nextNewline, whichever is closest 1601 var checkUpTo = nextNewline === -1 ? nextDelim : Math.min(nextDelim, nextNewline); 1602 var spacesBetweenQuoteAndDelimiter = extraSpaces(checkUpTo); 1603 1604 // Closing quote followed by delimiter or 'unnecessary spaces + delimiter' 1605 if (input.substr(quoteSearch + 1 + spacesBetweenQuoteAndDelimiter, delimLen) === delim) 1606 { 1607 row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); 1608 cursor = quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen; 1609 1610 // If char after following delimiter is not quoteChar, we find next quote char position 1611 if (input[quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen] !== quoteChar) 1612 { 1613 quoteSearch = input.indexOf(quoteChar, cursor); 1614 } 1615 nextDelim = input.indexOf(delim, cursor); 1616 nextNewline = input.indexOf(newline, cursor); 1617 break; 1618 } 1619 1620 var spacesBetweenQuoteAndNewLine = extraSpaces(nextNewline); 1621 1622 // Closing quote followed by newline or 'unnecessary spaces + newLine' 1623 if (input.substring(quoteSearch + 1 + spacesBetweenQuoteAndNewLine, quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen) === newline) 1624 { 1625 row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); 1626 saveRow(quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen); 1627 nextDelim = input.indexOf(delim, cursor); // because we may have skipped the nextDelim in the quoted field 1628 quoteSearch = input.indexOf(quoteChar, cursor); // we search for first quote in next line 1629 1630 if (stepIsFunction) 1631 { 1632 doStep(); 1633 if (aborted) 1634 return returnable(); 1635 } 1636 1637 if (preview && data.length >= preview) 1638 return returnable(true); 1639 1640 break; 1641 } 1642 1643 1644 // Checks for valid closing quotes are complete (escaped quotes or quote followed by EOF/delimiter/newline) -- assume these quotes are part of an invali
vendor: 5,882 bytes, lines 1644-1899
1644d text string 1645 errors.push({ 1646 type: 'Quotes', 1647 code: 'InvalidQuotes', 1648 message: 'Trailing quote on quoted field is malformed', 1649 row: data.length, // row has yet to be inserted 1650 index: cursor 1651 }); 1652 1653 quoteSearch++; 1654 continue; 1655 1656 } 1657 1658 continue; 1659 } 1660 1661 // Comment found at start of new line 1662 if (comments && row.length === 0 && input.substring(cursor, cursor + commentsLen) === comments) 1663 { 1664 if (nextNewline === -1) // Comment ends at EOF 1665 return returnable(); 1666 cursor = nextNewline + newlineLen; 1667 nextNewline = input.indexOf(newline, cursor); 1668 nextDelim = input.indexOf(delim, cursor); 1669 continue; 1670 } 1671 1672 // Next delimiter comes before next newline, so we've reached end of field 1673 if (nextDelim !== -1 && (nextDelim < nextNewline || nextNewline === -1)) 1674 { 1675 row.push(input.substring(cursor, nextDelim)); 1676 cursor = nextDelim + delimLen; 1677 // we look for next delimiter char 1678 nextDelim = input.indexOf(delim, cursor); 1679 continue; 1680 } 1681 1682 // End of row 1683 if (nextNewline !== -1) 1684 { 1685 row.push(input.substring(cursor, nextNewline)); 1686 saveRow(nextNewline + newlineLen); 1687 1688 if (stepIsFunction) 1689 { 1690 doStep(); 1691 if (aborted) 1692 return returnable(); 1693 } 1694 1695 if (preview && data.length >= preview) 1696 return returnable(true); 1697 1698 continue; 1699 } 1700 1701 break; 1702 } 1703 1704 1705 return finish(); 1706 1707 1708 function pushRow(row) 1709 { 1710 data.push(row); 1711 lastCursor = cursor; 1712 } 1713 1714 /** 1715 * checks if there are extra spaces after closing quote and given index without any text 1716 * if Yes, returns the number of spaces 1717 */ 1718 function extraSpaces(index) { 1719 var spaceLength = 0; 1720 if (index !== -1) { 1721 var textBetweenClosingQuoteAndIndex = input.substring(quoteSearch + 1, index); 1722 if (textBetweenClosingQuoteAndIndex && textBetweenClosingQuoteAndIndex.trim() === '') { 1723 spaceLength = textBetweenClosingQuoteAndIndex.length; 1724 } 1725 } 1726 return spaceLength; 1727 } 1728 1729 /** 1730 * Appends the remaining input from cursor to the end into 1731 * row, saves the row, calls step, and returns the results. 1732 */ 1733 function finish(value) 1734 { 1735 if (ignoreLastRow) 1736 return returnable(); 1737 if (typeof value === 'undefined') 1738 value = input.substring(cursor); 1739 row.push(value); 1740 cursor = inputLen; // important in case parsing is paused 1741 pushRow(row); 1742 if (stepIsFunction) 1743 doStep(); 1744 return returnable(); 1745 } 1746 1747 /** 1748 * Appends the current row to the results. It sets the cursor 1749 * to newCursor and finds the nextNewline. The caller should 1750 * take care to execute user's step function and check for 1751 * preview and end parsing if necessary. 1752 */ 1753 function saveRow(newCursor) 1754 { 1755 cursor = newCursor; 1756 pushRow(row); 1757 row = []; 1758 nextNewline = input.indexOf(newline, cursor); 1759 } 1760 1761 /** Returns an object with the results, errors, and meta. */ 1762 function returnable(stopped) 1763 { 1764 return { 1765 data: data, 1766 errors: errors, 1767 meta: { 1768 delimiter: delim, 1769 linebreak: newline, 1770 aborted: aborted, 1771 truncated: !!stopped, 1772 cursor: lastCursor + (baseIndex || 0) 1773 } 1774 }; 1775 } 1776 1777 /** Executes the user's step function and resets data & errors. */ 1778 function doStep() 1779 { 1780 step(returnable()); 1781 data = []; 1782 errors = []; 1783 } 1784 }; 1785 1786 /** Sets the abort flag */ 1787 this.abort = function() 1788 { 1789 aborted = true; 1790 }; 1791 1792 /** Gets the cursor position */ 1793 this.getCharIndex = function() 1794 { 1795 return cursor; 1796 }; 1797 } 1798 1799 1800 function newWorker() 1801 { 1802 if (!Papa.WORKERS_SUPPORTED) 1803 return false; 1804 1805 var workerUrl = getWorkerBlob(); 1806 var w = new global.Worker(workerUrl); 1807 w.onmessage = mainThreadReceivedMessage; 1808 w.id = workerIdCounter++; 1809 workers[w.id] = w; 1810 return w; 1811 } 1812 1813 /** Callback when main thread receives a message */ 1814 function mainThreadReceivedMessage(e) 1815 { 1816 var msg = e.data; 1817 var worker = workers[msg.workerId]; 1818 var aborted = false; 1819 1820 if (msg.error) 1821 worker.userError(msg.error, msg.file); 1822 else if (msg.results && msg.results.data) 1823 { 1824 var abort = function() { 1825 aborted = true; 1826 completeWorker(msg.workerId, { data: [], errors: [], meta: { aborted: true } }); 1827 }; 1828 1829 var handle = { 1830 abort: abort, 1831 pause: notImplemented, 1832 resume: notImplemented 1833 }; 1834 1835 if (isFunction(worker.userStep)) 1836 { 1837 for (var i = 0; i < msg.results.data.length; i++) 1838 { 1839 worker.userStep({ 1840 data: msg.results.data[i], 1841 errors: msg.results.errors, 1842 meta: msg.results.meta 1843 }, handle); 1844 if (aborted) 1845 break; 1846 } 1847 delete msg.results; // free memory ASAP 1848 } 1849 else if (isFunction(worker.userChunk)) 1850 { 1851 worker.userChunk(msg.results, handle, msg.file); 1852 delete msg.results; 1853 } 1854 } 1855 1856 if (msg.finished && !aborted) 1857 completeWorker(msg.workerId, msg.results); 1858 } 1859 1860 function completeWorker(workerId, results) { 1861 var worker = workers[workerId]; 1862 if (isFunction(worker.userComplete)) 1863 worker.userComplete(results); 1864 worker.terminate(); 1865 delete workers[workerId]; 1866 } 1867 1868 function notImplemented() { 1869 throw new Error('Not implemented.'); 1870 } 1871 1872 /** Callback when worker thread receives a message */ 1873 function workerThreadReceivedMessage(e) 1874 { 1875 var msg = e.data; 1876 1877 if (typeof Papa.WORKER_ID === 'undefined' && msg) 1878 Papa.WORKER_ID = msg.workerId; 1879 1880 if (typeof msg.input === 'string') 1881 { 1882 global.postMessage({ 1883 workerId: Papa.WORKER_ID, 1884 results: Papa.parse(msg.input, msg.config), 1885 finished: true 1886 }); 1887 } 1888 else if ((global.File && msg.input instanceof File) || msg.input instanceof Object) // thank you, Safari (see issue #106) 1889 { 1890 var results = Papa.parse(msg.input, msg.config); 1891 if (results) 1892 global.postMessage({ 1893 workerId: Papa.WORKER_ID, 1894 results: results, 1895 finished: true 1896 }); 1897 } 1898 } 1899
1900 /** Makes a deep copy of an array or object (mostly) */ 1901 function copy(obj) 1902 { 1903 if (typeof obj !== 'object' || obj === null) 1904 return obj; 1905 var cpy = Array.isArray(obj) ? [] : {}; 1906 for (var key in obj) 1907 cpy[key] = copy(obj[key]); 1908 return cpy; 1909 } 1910 1911 function bindFunction(f, self) 1912 { 1913 return function() { f.apply(self, arguments); }; 1914 } 1915 1916 function isFunction(func) 1917 { 1918 return typeof func === 'function'; 1919 } 1920 1921 return Papa; 1922}));
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.