1import { JSONRepairError } from '../utils/JSONRepairError.js'; 2import { codeAsterisk, codeBackslash, codeCloseParenthesis, codeClosingBrace, codeClosingBracket, codeColon, codeComma, codeDot, codeDoubleQuote, codeLowercaseE, codeMinus, codeNewline, codeOpeningBrace, codeOpeningBracket, codeOpenParenthesis, codePlus, codeSemicolon, codeSlash, codeUppercaseE, endsWithCommaOrNewline, insertBeforeLastWhitespace, isControlCharacter, isDelimiter, isDelimiterExceptSlash, isDigit, isDoubleQuote, isDoubleQuoteLike, isFunctionName, isHex, isQuote, isSingleQuote, isSingleQuoteLike, isSpecialWhitespace, isStartOfValue, isValidStringCharacter, isWhitespace, removeAtIndex, stripLastOccurrence } from '../utils/stringUtils.js'; 3const controlCharacters = { 4 '\b': '\\b', 5 '\f': '\\f', 6 '\n': '\\n', 7 '\r': '\\r', 8 '\t': '\\t' 9}; 10 11// map with all escape characters 12const escapeCharacters = { 13 '"': '"', 14 '\\': '\\', 15 '/': '/', 16 b: '\b', 17 f: '\f', 18 n: '\n', 19 r: '\r', 20 t: '\t' 21 // note that \u is handled separately in parseString() 22}; 23 24/** 25 * Repair a string containing an invalid JSON document. 26 * For example changes JavaScript notation into JSON notation. 27 * 28 * Example: 29 * 30 * try { 31 * const json = "{name: 'John'}" 32 * const repaired = jsonrepair(json) 33 * console.log(repaired) 34 * // '{"name": "John"}' 35 * } catch (err) { 36 * console.error(err) 37 * } 38 * 39 */ 40export function jsonrepair(text) { 41 let i = 0; // current index in text 42 let output = ''; // generated output 43 44 const processed = parseValue(); 45 if (!processed) { 46 throwUnexpectedEnd(); 47 } 48 const processedComma = parseCharacter(codeComma); 49 if (processedComma) { 50 parseWhitespaceAndSkipComments(); 51 } 52 if (isStartOfValue(text[i]) && endsWithCommaOrNewline(output)) { 53 // start of a new value after end of the root level object: looks like 54 // newline delimited JSON -> turn into a root level array 55 if (!processedComma) { 56 // repair missing comma 57 output = insertBeforeLastWhitespace(output, ','); 58 } 59 parseNewlineDelimitedJSON(); 60 } else if (processedComma) { 61 // repair: remove trailing comma 62 output = stripLastOccurrence(output, ','); 63 } 64 65 // repair redundant end quotes 66 while (text.charCodeAt(i) === codeClosingBrace || text.charCodeAt(i) === codeClosingBracket) { 67 i++; 68 parseWhitespaceAndSkipComments(); 69 } 70 if (i >= text.length) { 71 // reached the end of the document properly 72 return output; 73 } 74 throwUnexpectedCharacter(); 75 function parseValue() { 76 parseWhitespaceAndSkipComments(); 77 const processed = parseObject() || parseArray() || parseString() || parseNumber() || parseKeywords() || parseUnquotedString(); 78 parseWhitespaceAndSkipComments(); 79 return processed; 80 } 81 function parseWhitespaceAndSkipComments() { 82 const start = i; 83 let changed = parseWhitespace(); 84 do { 85 changed = parseComment(); 86 if (changed) { 87 changed = parseWhitespace(); 88 } 89 } while (changed); 90 return i > start; 91 } 92 function parseWhitespace() { 93 let whitespace = ''; 94 let normal; 95 while ((normal = isWhitespace(text.charCodeAt(i))) || isSpecialWhitespace(text.charCodeAt(i))) { 96 if (normal) { 97 whitespace += text[i]; 98 } else { 99 // repair special whitespace 100 whitespace += ' '; 101 } 102 i++; 103 } 104 if (whitespace.length > 0) { 105 output += whitespace; 106 return true; 107 } 108 return false; 109 } 110 function parseComment() { 111 // find a block comment '/* ... */' 112 if (text.charCodeAt(i) === codeSlash && text.charCodeAt(i + 1) === codeAsterisk) { 113 // repair block comment by skipping it 114 while (i < text.length && !atEndOfBlockComment(text, i)) { 115 i++; 116 } 117 i += 2; 118 return true; 119 } 120 121 // find a line comment '// ...' 122 if (text.charCodeAt(i) === codeSlash && text.charCodeAt(i + 1) === codeSlash) { 123 // repair line comment by skipping it 124 while (i < text.length && text.charCodeAt(i) !== codeNewline) { 125 i++; 126 } 127 return true; 128 } 129 return false; 130 } 131 function parseCharacter(code) { 132 if (text.charCodeAt(i) === code) { 133 output += text[i]; 134 i++; 135 return true; 136 } 137 return false;
138 } 139 function skipCharacter(code) { 140 if (text.charCodeAt(i) === code) { 141 i++; 142 return true; 143 } 144 return false; 145 } 146 function skipEscapeCharacter() { 147 return skipCharacter(codeBackslash); 148 } 149 150 /** 151 * Skip ellipsis like "[1,2,3,...]" or "[1,2,3,...,9]" or "[...,7,8,9]" 152 * or a similar construct in objects. 153 */ 154 function skipEllipsis() { 155 parseWhitespaceAndSkipComments(); 156 if (text.charCodeAt(i) === codeDot && text.charCodeAt(i + 1) === codeDot && text.charCodeAt(i + 2) === codeDot) { 157 // repair: remove the ellipsis (three dots) and optionally a comma 158 i += 3; 159 parseWhitespaceAndSkipComments(); 160 skipCharacter(codeComma); 161 return true; 162 } 163 return false; 164 } 165 166 /** 167 * Parse an object like '{"key": "value"}' 168 */ 169 function parseObject() { 170 if (text.charCodeAt(i) === codeOpeningBrace) { 171 output += '{'; 172 i++; 173 parseWhitespaceAndSkipComments(); 174 175 // repair: skip leading comma like in {, message: "hi"} 176 if (skipCharacter(codeComma)) { 177 parseWhitespaceAndSkipComments(); 178 } 179 let initial = true; 180 while (i < text.length && text.charCodeAt(i) !== codeClosingBrace) { 181 let processedComma; 182 if (!initial) { 183 processedComma = parseCharacter(codeComma); 184 if (!processedComma) { 185 // repair missing comma 186 output = insertBeforeLastWhitespace(output, ','); 187 } 188 parseWhitespaceAndSkipComments(); 189 } else { 190 processedComma = true; 191 initial = false; 192 } 193 skipEllipsis(); 194 const processedKey = parseString() || parseUnquotedString(); 195 if (!processedKey) { 196 if (text.charCodeAt(i) === codeClosingBrace || text.charCodeAt(i) === codeOpeningBrace || text.charCodeAt(i) === codeClosingBracket || text.charCodeAt(i) === codeOpeningBracket || text[i] === undefined) { 197 // repair trailing comma 198 output = stripLastOccurrence(output, ','); 199 } else { 200 throwObjectKeyExpected(); 201 } 202 break; 203 } 204 parseWhitespaceAndSkipComments(); 205 const processedColon = parseCharacter(codeColon); 206 const truncatedText = i >= text.length; 207 if (!processedColon) { 208 if (isStartOfValue(text[i]) || truncatedText) { 209 // repair missing colon 210 output = insertBeforeLastWhitespace(output, ':'); 211 } else { 212 throwColonExpected(); 213 } 214 } 215 const processedValue = parseValue(); 216 if (!processedValue) { 217 if (processedColon || truncatedText) { 218 // repair missing object value 219 output += 'null'; 220 } else { 221 throwColonExpected(); 222 } 223 } 224 } 225 if (text.charCodeAt(i) === codeClosingBrace) { 226 output += '}'; 227 i++; 228 } else { 229 // repair missing end bracket 230 output = insertBeforeLastWhitespace(output, '}'); 231 } 232 return true; 233 } 234 return false; 235 } 236 237 /** 238 * Parse an array like '["item1", "item2", ...]' 239 */ 240 function parseArray() { 241 if (text.charCodeAt(i) === codeOpeningBracket) { 242 output += '['; 243 i++; 244 parseWhitespaceAndSkipComments(); 245 246 // repair: skip leading comma like in [,1,2,3] 247 if (skipCharacter(codeComma)) { 248 parseWhitespaceAndSkipComments(); 249 } 250 let initial = true; 251 while (i < text.length && text.charCodeAt(i) !== codeClosingBracket) { 252 if (!initial) { 253 const processedComma = parseCharacter(codeComma); 254 if (!processedComma) { 255 // repair missing comma 256 output = insertBeforeLastWhitespace(output, ','); 257 } 258 } else { 259 initial = false; 260 } 261 skipEllipsis(); 262 const processedValue = parseValue(); 263 if (!processedValue) { 264 // repair trailing comma 265 output = stripLastOccurrence(output, ','); 266 break; 267 } 268 } 269 if (text.charCodeAt(i) === codeClosingBracket) { 270 output += ']'; 271 i++; 272 } else { 273 // repair missing closing array bracket 274 output = insertBeforeLastWhitespace(output, ']'); 275 } 276 return true; 277 } 278 return false;
279 } 280 281 /** 282 * Parse and repair Newline Delimited JSON (NDJSON): 283 * multiple JSON objects separated by a newline character 284 */ 285 function parseNewlineDelimitedJSON() { 286 // repair NDJSON 287 let initial = true; 288 let processedValue = true; 289 while (processedValue) { 290 if (!initial) { 291 // parse optional comma, insert when missing 292 const processedComma = parseCharacter(codeComma); 293 if (!processedComma) { 294 // repair: add missing comma 295 output = insertBeforeLastWhitespace(output, ','); 296 } 297 } else { 298 initial = false; 299 } 300 processedValue = parseValue(); 301 } 302 if (!processedValue) { 303 // repair: remove trailing comma 304 output = stripLastOccurrence(output, ','); 305 } 306 307 // repair: wrap the output inside array brackets 308 output = "[\n".concat(output, "\n]"); 309 } 310 311 /** 312 * Parse a string enclosed by double quotes "...". Can contain escaped quotes 313 * Repair strings enclosed in single quotes or special quotes 314 * Repair an escaped string 315 * 316 * The function can run in two stages: 317 * - First, it assumes the string has a valid end quote 318 * - If it turns out that the string does not have a valid end quote followed 319 * by a delimiter (which should be the case), the function runs again in a 320 * more conservative way, stopping the string at the first next delimiter 321 * and fixing the string by inserting a quote there. 322 */ 323 function parseString() { 324 let stopAtDelimiter = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : false; 325 let skipEscapeChars = text.charCodeAt(i) === codeBackslash; 326 if (skipEscapeChars) { 327 // repair: remove the first escape character 328 i++; 329 skipEscapeChars = true; 330 } 331 if (isQuote(text.charCodeAt(i))) { 332 // double quotes are correct JSON, 333 // single quotes come from JavaScript for example, we assume it will have a correct single end quote too 334 // otherwise, we will match any double-quote-like start with a double-quote-like end, 335 // or any single-quote-like start with a single-quote-like end 336 const isEndQuote = isDoubleQuote(text.charCodeAt(i)) ? isDoubleQuote : isSingleQuote(text.charCodeAt(i)) ? isSingleQuote : isSingleQuoteLike(text.charCodeAt(i)) ? isSingleQuoteLike : isDoubleQuoteLike; 337 const iBefore = i; 338 const oBefore = output.length; 339 let str = '"'; 340 i++; 341 while (true) { 342 if (i >= text.length) { 343 // end of text, we are missing an end quote 344 345 const iPrev = prevNonWhitespaceIndex(i - 1); 346 if (!stopAtDelimiter && isDelimiter(text.charAt(iPrev))) { 347 // if the text ends with a delimiter, like ["hello], 348 // so the missing end quote should be inserted before this delimiter 349 // retry parsing the string, stopping at the first next delimiter 350 i = iBefore; 351 output = output.substring(0, oBefore); 352 return parseString(true); 353 } 354 355 // repair missing quote 356 str = insertBeforeLastWhitespace(str, '"'); 357 output += str; 358 return true; 359 } else if (isEndQuote(text.charCodeAt(i))) { 360 // end quote 361 // let us check what is before and after the quote to verify whether this is a legit end quote 362 const iQuote = i; 363 const oQuote = str.length; 364 str += '"'; 365 i++; 366 output += str; 367 parseWhitespaceAndSkipComments(); 368 if (stopAtDelimiter || i >= text.length || isDelimiter(text.charAt(i)) || isQuote(text.charCodeAt(i)) || isDigit(text.charCodeAt(i))) { 369 // The quote is followed by the end of the text, a delimiter, or a next value 370 // so the quote is indeed the end of the string 371 parseConcatenatedString(); 372 return true; 373 } 374 if (isDelimiter(text.charAt(prevNonWhitespaceIndex(iQuote - 1)))) { 375 // This is not the right end quote: it is preceded by a delimiter, 376 // and NOT followed by a delimiter. So, there is an end quote missing 377 // parse the string again and then stop at the first next delimiter 378 i = iBefore; 379 output = output.substring(0, oBefore); 380 return parseString(true); 381 } 382 383 // revert to right after the quote but before any whitespace, and continue parsing the string 384 output = output.substring(0, oBefore); 385 i = iQuote + 1; 386 387 // repair unescaped quote 388 str = str.substring(0, oQuote) + '\\' + str.substring(oQuote); 389 } else if (stopAtDelimiter && isDelimiter(text[i])) { 390 // we're in the mode to stop the string at the first delimiter 391 // because there is an end quote missing 392 393 // repair missing quote 394 str = insertBeforeLastWhitespace(str, '"'); 395 output += str; 396 parseConcatenatedString(); 397 return true; 398 } else if (text.charCodeAt(i) === codeBackslash) { 399 // handle escaped content like \n or \u2605 400 const char = text.charAt(i + 1); 401 const escapeChar = escapeCharacters[char]; 402 if (escapeChar !== undefined) { 403 str += text.slice(i, i + 2); 404 i += 2; 405 } else if (char === 'u') { 406 let j = 2; 407 while (j < 6 && isHex(text.charCodeAt(i + j))) { 408 j++; 409 } 410 if (j === 6) { 411 str += text.slice(i, i + 6); 412 i += 6; 413 } else if (i + j >= text.length) { 414 // repair invalid or truncated unicode char at the end of the text 415 // by removing the unicode char and ending the string here 416 i = text.length; 417 } else { 418 throwInvalidUnicodeCharacter(); 419 } 420 } else { 421 // repair invalid escape character: remove it 422 str += char; 423 i += 2; 424 } 425 } else { 426 // handle regular characters 427 const char = text.charAt(i); 428 const code = text.charCodeAt(i); 429 if (code === codeDoubleQuote && text.charCodeAt(i - 1) !== codeBackslash) { 430 // repair unescaped double quote 431 str += '\\' + char; 432 i++; 433 } else if (isControlCharacter(code)) { 434 // unescaped control character 435 str += controlCharacters[char]; 436 i++; 437 } else { 438 if (!isValidStringCharacter(code)) { 439 throwInvalidCharacter(char); 440 } 441 str += char; 442 i++; 443 } 444 } 445 if (skipEscapeChars) { 446 // repair: skipped escape character (nothing to do) 447 skipEscapeCharacter(); 448 } 449 } 450 } 451 return false;
452 } 453 454 /** 455 * Repair concatenated strings like "hello" + "world", change this into "helloworld" 456 */ 457 function parseConcatenatedString() { 458 let processed = false; 459 parseWhitespaceAndSkipComments(); 460 while (text.charCodeAt(i) === codePlus) { 461 processed = true; 462 i++; 463 parseWhitespaceAndSkipComments(); 464 465 // repair: remove the end quote of the first string 466 output = stripLastOccurrence(output, '"', true); 467 const start = output.length; 468 const parsedStr = parseString(); 469 if (parsedStr) { 470 // repair: remove the start quote of the second string 471 output = removeAtIndex(output, start, 1); 472 } else { 473 // repair: remove the + because it is not followed by a string 474 output = insertBeforeLastWhitespace(output, '"'); 475 } 476 } 477 return processed; 478 } 479 480 /** 481 * Parse a number like 2.4 or 2.4e6 482 */ 483 function parseNumber() { 484 const start = i; 485 if (text.charCodeAt(i) === codeMinus) { 486 i++; 487 if (atEndOfNumber()) { 488 repairNumberEndingWithNumericSymbol(start); 489 return true; 490 } 491 if (!isDigit(text.charCodeAt(i))) { 492 i = start; 493 return false; 494 } 495 } 496 497 // Note that in JSON leading zeros like "00789" are not allowed. 498 // We will allow all leading zeros here though and at the end of parseNumber 499 // check against trailing zeros and repair that if needed. 500 // Leading zeros can have meaning, so we should not clear them. 501 while (isDigit(text.charCodeAt(i))) { 502 i++; 503 } 504 if (text.charCodeAt(i) === codeDot) { 505 i++; 506 if (atEndOfNumber()) { 507 repairNumberEndingWithNumericSymbol(start); 508 return true; 509 } 510 if (!isDigit(text.charCodeAt(i))) { 511 i = start; 512 return false; 513 } 514 while (isDigit(text.charCodeAt(i))) { 515 i++; 516 } 517 } 518 if (text.charCodeAt(i) === codeLowercaseE || text.charCodeAt(i) === codeUppercaseE) { 519 i++; 520 if (text.charCodeAt(i) === codeMinus || text.charCodeAt(i) === codePlus) { 521 i++; 522 } 523 if (atEndOfNumber()) { 524 repairNumberEndingWithNumericSymbol(start); 525 return true; 526 } 527 if (!isDigit(text.charCodeAt(i))) { 528 i = start; 529 return false; 530 } 531 while (isDigit(text.charCodeAt(i))) { 532 i++; 533 } 534 } 535 536 // if we're not at the end of the number by this point, allow this to be parsed as another type 537 if (!atEndOfNumber()) { 538 i = start; 539 return false; 540 } 541 if (i > start) { 542 // repair a number with leading zeros like "00789" 543 const num = text.slice(start, i); 544 const hasInvalidLeadingZero = /^0\d/.test(num); 545 output += hasInvalidLeadingZero ? "\"".concat(num, "\"") : num; 546 return true; 547 } 548 return false; 549 } 550 551 /** 552 * Parse keywords true, false, null 553 * Repair Python keywords True, False, None 554 */ 555 function parseKeywords() { 556 return parseKeyword('true', 'true') || parseKeyword('false', 'false') || parseKeyword('null', 'null') || 557 // repair Python keywords True, False, None 558 parseKeyword('True', 'true') || parseKeyword('False', 'false') || parseKeyword('None', 'null'); 559 } 560 function parseKeyword(name, value) { 561 if (text.slice(i, i + name.length) === name) { 562 output += value; 563 i += name.length; 564 return true; 565 } 566 return false; 567 } 568 569 /** 570 * Repair an unquoted string by adding quotes around it 571 * Repair a MongoDB function call like NumberLong("2") 572 * Repair a JSONP function call like callback({...}); 573 */ 574 function parseUnquotedString() { 575 // note that the symbol can end with whitespaces: we stop at the next delimiter 576 // also, note that we allow strings to contain a slash / in order to support repairing regular expressions 577 const start = i; 578 while (i < text.length && !isDelimiterExceptSlash(text[i]) && !isQuote(text.charCodeAt(i))) { 579 i++; 580 } 581 if (i > start) { 582 if (text.charCodeAt(i) === codeOpenParenthesis && isFunctionName(text.slice(start, i).trim())) { 583 // repair a MongoDB function call like NumberLong("2") 584 // repair a JSONP function call like callback({...}); 585 i++; 586 parseValue(); 587 if (text.charCodeAt(i) === codeCloseParenthesis) { 588 // repair: skip close bracket of function call 589 i++; 590 if (text.charCodeAt(i) === codeSemicolon) { 591 // repair: skip semicolon after JSONP call 592 i++; 593 } 594 } 595 return true; 596 } else { 597 // repair unquoted string 598 // also, repair undefined into null 599 600 // first, go back to prevent getting trailing whitespace
600s in the string 601 while (isWhitespace(text.charCodeAt(i - 1)) && i > 0) { 602 i--; 603 } 604 const symbol = text.slice(start, i); 605 output += symbol === 'undefined' ? 'null' : JSON.stringify(symbol); 606 if (text.charCodeAt(i) === codeDoubleQuote) { 607 // we had a missing start quote, but now we encountered the end quote, so we can skip that one 608 i++; 609 } 610 return true; 611 } 612 } 613 } 614 function prevNonWhitespaceIndex(start) { 615 let prev = start; 616 while (prev > 0 && isWhitespace(text.charCodeAt(prev))) { 617 prev--; 618 } 619 return prev; 620 } 621 function atEndOfNumber() { 622 return i >= text.length || isDelimiter(text[i]) || isWhitespace(text.charCodeAt(i)); 623 } 624 function repairNumberEndingWithNumericSymbol(start) { 625 // repair numbers cut off at the end 626 // this will only be called when we end after a '.', '-', or 'e' and does not 627 // change the number more than it needs to make it valid JSON 628 output += text.slice(start, i) + '0'; 629 } 630 function throwInvalidCharacter(char) { 631 throw new JSONRepairError('Invalid character ' + JSON.stringify(char), i); 632 } 633 function throwUnexpectedCharacter() { 634 throw new JSONRepairError('Unexpected character ' + JSON.stringify(text[i]), i); 635 } 636 function throwUnexpectedEnd() { 637 throw new JSONRepairError('Unexpected end of json string', text.length); 638 } 639 function throwObjectKeyExpected() { 640 throw new JSONRepairError('Object key expected', i); 641 } 642 function throwColonExpected() { 643 throw new JSONRepairError('Colon expected', i); 644 } 645 function throwInvalidUnicodeCharacter() { 646 const chars = text.slice(i, i + 6); 647 throw new JSONRepairError("Invalid unicode character \"".concat(chars, "\""), i); 648 } 649} 650function atEndOfBlockComment(text, i) { 651 return text[i] === '*' && text[i + 1] === '/'; 652} 653//# sourceMappingURL=jsonrepair.js.map
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.