1/* 2 Copyright (c) 2022 Gildas Lormeau. All rights reserved. 3 4 Redistribution and use in source and binary forms, with or without 5 modification, are permitted provided that the following conditions are met: 6 7 1. Redistributions of source code must retain the above copyright notice, 8 this list of conditions and the following disclaimer. 9 10 2. Redistributions in binary form must reproduce the above copyright 11 notice, this list of conditions and the following disclaimer in 12 the documentation and/or other materials provided with the distribution. 13 14 3. The names of the authors may not be used to endorse or promote products 15 derived from this software without specific prior written permission. 16 17 THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, 18 INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 19 FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, 20 INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, 21 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, 23 OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 24 LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 25 NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29/* global Blob, atob, btoa, XMLHttpRequest, URL, fetch, ReadableStream, WritableStream, FileReader, TransformStream, Response */ 30// deno-lint-ignore-file no-this-alias 31 32import { 33 UNDEFINED_VALUE, 34 FUNCTION_TYPE 35} from "./constants.js"; 36import { getConfiguration } from "./configuration.js"; 37 38const ERR_HTTP_STATUS = "HTTP error "; 39const ERR_HTTP_RANGE = "HTTP Range not supported"; 40const ERR_ITERATOR_COMPLETED_TOO_SOON = "Writer iterator completed too soon"; 41 42const CONTENT_TYPE_TEXT_PLAIN = "text/plain"; 43const HTTP_HEADER_CONTENT_LENGTH = "Content-Length"; 44const HTTP_HEADER_CONTENT_RANGE = "Content-Range"; 45const HTTP_HEADER_ACCEPT_RANGES = "Accept-Ranges"; 46const HTTP_HEADER_RANGE = "Range"; 47const HTTP_HEADER_CONTENT_TYPE = "Content-Type"; 48const HTTP_METHOD_HEAD = "HEAD"; 49const HTTP_METHOD_GET = "GET"; 50const HTTP_RANGE_UNIT = "bytes"; 51const DEFAULT_CHUNK_SIZE = 64 * 1024; 52 53const PROPERTY_NAME_WRITABLE = "writable"; 54 55class Stream { 56 57 constructor() { 58 this.size = 0; 59 } 60
61 init() { 62 this.initialized = true; 63 } 64} 65 66class Reader extends Stream { 67 68 get readable() { 69 const reader = this; 70 const { chunkSize = DEFAULT_CHUNK_SIZE } = reader; 71 const readable = new ReadableStream({ 72 start() { 73 this.chunkOffset = 0; 74 }, 75 async pull(controller) { 76 const { offset = 0, size, diskNumberStart } = readable; 77 const { chunkOffset } = this; 78 controller.enqueue(await readUint8Array(reader, offset + chunkOffset, Math.min(chunkSize, size - chunkOffset), diskNumberStart)); 79 if (chunkOffset + chunkSize > size) { 80 controller.close(); 81 } else { 82 this.chunkOffset += chunkSize; 83 } 84 } 85 }); 86 return readable; 87 } 88} 89 90class Writer extends Stream { 91 92 constructor() { 93 super(); 94 const writer = this; 95 const writable = new WritableStream({ 96 write(chunk) { 97 return writer.writeUint8Array(chunk); 98 } 99 }); 100 Object.defineProperty(writer, PROPERTY_NAME_WRITABLE, { 101 get() { 102 return writable; 103 } 104 }); 105 } 106 107 writeUint8Array() { 108 // abstract 109 } 110} 111 112class Data64URIReader extends Reader { 113 114 constructor(dataURI) { 115 super(); 116 let dataEnd = dataURI.length; 117 while (dataURI.charAt(dataEnd - 1) == "=") { 118 dataEnd--; 119 } 120 const dataStart = dataURI.indexOf(",") + 1; 121 Object.assign(this, { 122 dataURI, 123 dataStart, 124 size: Math.floor((dataEnd - dataStart) * 0.75) 125 }); 126 } 127 128 readUint8Array(offset, length) { 129 const { 130 dataStart, 131 dataURI 132 } = this; 133 const dataArray = new Uint8Array(length); 134 const start = Math.floor(offset / 3) * 4; 135 const bytes = atob(dataURI.substring(start + dataStart, Math.ceil((offset + length) / 3) * 4 + dataStart)); 136 const delta = offset - Math.floor(start / 4) * 3; 137 for (let indexByte = delta; indexByte < delta + length; indexByte++) { 138 dataArray[indexByte - delta] = bytes.charCodeAt(indexByte); 139 } 140 return dataArray; 141 } 142} 143 144class Data64URIWriter extends Writer { 145 146 constructor(contentType) { 147 super(); 148 Object.assign(this, { 149 data: "data:" + (contentType || "") + ";base64,", 150 pending: [] 151 }); 152 } 153 154 writeUint8Array(array) { 155 const writer = this; 156 let indexArray = 0; 157 let dataString = writer.pending; 158 const delta = writer.pending.length; 159 writer.pending = ""; 160 for (indexArray = 0; indexArray < (Math.floor((delta + array.length) / 3) * 3) - delta; indexArray++) { 161 dataString += String.fromCharCode(array[indexArray]); 162 } 163 for (; indexArray < array.length; indexArray++) { 164 writer.pending += String.fromCharCode(array[indexArray]); 165 } 166 if (dataString.length > 2) { 167 writer.data += btoa(dataString); 168 } else { 169 writer.pending = dataString; 170 } 171 } 172 173 getData() { 174 return this.data + btoa(this.pending); 175 } 176} 177 178class BlobReader extends Reader { 179 180 constructor(blob) { 181 super(); 182 Object.assign(this, { 183 blob, 184 size: blob.size 185 }); 186 } 187 188 async readUint8Array(offset, length) { 189 const reader = this; 190 const offsetEnd = offset + length; 191 const blob = offset || offsetEnd < reader.size ? reader.blob.slice(offset, offsetEnd) : reader.blob; 192 return new Uint8Array(await blob.arrayBuffer()); 193 } 194} 195 196class BlobWriter extends Stream { 197 198 constructor(contentType) { 199 super(); 200 const writer = this; 201 const transformStream = new TransformStream(); 202 const headers = []; 203 if (contentType) { 204 headers.push([HTTP_HEADER_CONTENT_TYPE, contentType]); 205 } 206 Object.defineProperty(writer, PROPERTY_NAME_WRITABLE, { 207 get() { 208 return transformStream.writable; 209 } 210 }); 211 writer.blob = new Response(transformStream.readable, { headers }).blob(); 212 } 213 214 getData() { 215 return this.blob; 216 } 217} 218 219class TextReader extends BlobReader { 220 221 constructor(text) { 222 super(new Blob([text], { type: CONTENT_TYPE_TEXT_PLAIN })); 223 } 224} 225 226class TextWriter extends BlobWriter { 227 228 constructor(encoding) { 229 super(encoding); 230 Object.assign(this, { 231 encoding, 232 utf8: !encoding || encoding.toLowerCase() == "utf-8" 233 }); 234 } 235 236 async getData() { 237 const { 238 encoding, 239 utf8 240 } = this; 241 const blob = await super.getData(); 242 if (blob.text && utf8) { 243 return blob.text(); 244 } else { 245 const reader = new FileReader(); 246 return new Promise((resolve, reject) => { 247 Object.assign(reader, { 248 onload: ({ target }) => resolve(target.result), 249 onerror: () => reject(reader.error) 250 }); 251 reader.readAsText(blob, encoding); 252 }); 253 } 254 } 255} 256 257class FetchReader extends Reader { 258 259 constructor(url, options) { 260 super(); 261 createHtpReader(this, url, options); 262 } 263
264 async init() { 265 await initHttpReader(this, sendFetchRequest, getFetchRequestData); 266 super.init(); 267 } 268 269 readUint8Array(index, length) { 270 return readUint8ArrayHttpReader(this, index, length, sendFetchRequest, getFetchRequestData); 271 } 272} 273 274class XHRReader extends Reader { 275 276 constructor(url, options) { 277 super(); 278 createHtpReader(this, url, options); 279 } 280 281 async init() { 282 await initHttpReader(this, sendXMLHttpRequest, getXMLHttpRequestData); 283 super.init(); 284 } 285 286 readUint8Array(index, length) { 287 return readUint8ArrayHttpReader(this, index, length, sendXMLHttpRequest, getXMLHttpRequestData); 288 } 289} 290 291function createHtpReader(httpReader, url, options) { 292 const { 293 preventHeadRequest, 294 useRangeHeader, 295 forceRangeRequests 296 } = options; 297 options = Object.assign({}, options); 298 delete options.preventHeadRequest; 299 delete options.useRangeHeader; 300 delete options.forceRangeRequests; 301 delete options.useXHR; 302 Object.assign(httpReader, { 303 url, 304 options, 305 preventHeadRequest, 306 useRangeHeader, 307 forceRangeRequests 308 }); 309} 310 311async function initHttpReader(httpReader, sendRequest, getRequestData) { 312 const { 313 url, 314 useRangeHeader, 315 forceRangeRequests 316 } = httpReader; 317 if (isHttpFamily(url) && (useRangeHeader || forceRangeRequests)) { 318 const { headers } = await sendRequest(HTTP_METHOD_GET, httpReader, getRangeHeaders(httpReader)); 319 if (!forceRangeRequests && headers.get(HTTP_HEADER_ACCEPT_RANGES) != HTTP_RANGE_UNIT) { 320 throw new Error(ERR_HTTP_RANGE); 321 } else { 322 let contentSize; 323 const contentRangeHeader = headers.get(HTTP_HEADER_CONTENT_RANGE); 324 if (contentRangeHeader) { 325 const splitHeader = contentRangeHeader.trim().split(/\s*\/\s*/); 326 if (splitHeader.length) { 327 const headerValue = splitHeader[1]; 328 if (headerValue && headerValue != "*") { 329 contentSize = Number(headerValue); 330 } 331 } 332 } 333 if (contentSize === UNDEFINED_VALUE) { 334 await getContentLength(httpReader, sendRequest, getRequestData); 335 } else { 336 httpReader.size = contentSize; 337 } 338 } 339 } else { 340 await getContentLength(httpReader, sendRequest, getRequestData); 341 } 342} 343 344async function readUint8ArrayHttpReader(httpReader, index, length, sendRequest, getRequestData) { 345 const { 346 useRangeHeader, 347 forceRangeRequests, 348 options 349 } = httpReader; 350 if (useRangeHeader || forceRangeRequests) { 351 const response = await sendRequest(HTTP_METHOD_GET, httpReader, getRangeHeaders(httpReader, index, length)); 352 if (response.status != 206) { 353 throw new Error(ERR_HTTP_RANGE); 354 } 355 return new Uint8Array(await response.arrayBuffer()); 356 } else { 357 const { data } = httpReader; 358 if (!data) { 359 await getRequestData(httpReader, options); 360 } 361 return new Uint8Array(httpReader.data.subarray(index, index + length)); 362 } 363} 364 365function getRangeHeaders(httpReader, index = 0, length = 1) { 366 return Object.assign({}, getHeaders(httpReader), { [HTTP_HEADER_RANGE]: HTTP_RANGE_UNIT + "=" + index + "-" + (index + length - 1) }); 367} 368 369function getHeaders({ options }) { 370 const { headers } = options; 371 if (headers) { 372 if (Symbol.iterator in headers) { 373 return Object.fromEntries(headers); 374 } else { 375 return headers; 376 } 377 } 378} 379 380async function getFetchRequestData(httpReader) { 381 await getRequestData(httpReader, sendFetchRequest); 382} 383 384async function getXMLHttpRequestData(httpReader) { 385 await getRequestData(httpReader, sendXMLHttpRequest); 386} 387 388async function getRequestData(httpReader, sendRequest) { 389 const response = await sendRequest(HTTP_METHOD_GET, httpReader, getHeaders(httpReader)); 390 httpReader.data = new Uint8Array(await response.arrayBuffer()); 391 if (!httpReader.size) { 392 httpReader.size = httpReader.data.length; 393 } 394} 395 396async function getContentLength(httpReader, sendRequest, getRequestData) { 397 if (httpReader.preventHeadRequest) { 398 await getRequestData(httpReader, httpReader.options); 399 } else { 400 const response = await sendRequest(HTTP_METHOD_HEAD, httpReader, getHeaders(httpReader)); 401 const contentLength = response.headers.get(HTTP_HEADER_CONTENT_LENGTH); 402 if (contentLength) { 403 httpReader.size = Number(contentLength); 404 } else { 405 await getRequestData(httpReader, httpReader.options); 406 } 407 } 408} 409 410async function sendFetchRequest(method, { options, url }, headers) { 411 const response = await fetch(url, Object.assign({}, options, { method, headers })); 412 if (response.status < 400) { 413 return response; 414 } else { 415 throw response.status == 416 ? new Error(ERR_HTTP_RANGE) : new Error(ERR_HTTP_STATUS + (response.statusText || response.status)); 416 } 417} 418 419function sendXMLHttpRequest(method, { url }, headers) { 420 return new Promise((resolve, reject) => { 421 const request = new XMLHttpRequest(); 422 request.addEventListener("load", () => { 423 if (request.status < 400) { 424 const headers = []; 425 request.getAllResponseHeaders().trim().split(/[\r\n]+/).forEac
425h(header => { 426 const splitHeader = header.trim().split(/\s*:\s*/); 427 splitHeader[0] = splitHeader[0].trim().replace(/^[a-z]|-[a-z]/g, value => value.toUpperCase()); 428 headers.push(splitHeader); 429 }); 430 resolve({ 431 status: request.status, 432 arrayBuffer: () => request.response, 433 headers: new Map(headers) 434 }); 435 } else { 436 reject(request.status == 416 ? new Error(ERR_HTTP_RANGE) : new Error(ERR_HTTP_STATUS + (request.statusText || request.status))); 437 } 438 }, false); 439 request.addEventListener("error", event => reject(event.detail.error), false); 440 request.open(method, url); 441 if (headers) { 442 for (const entry of Object.entries(headers)) { 443 request.setRequestHeader(entry[0], entry[1]); 444 } 445 } 446 request.responseType = "arraybuffer"; 447 request.send(); 448 }); 449} 450 451class HttpReader extends Reader { 452 453 constructor(url, options = {}) { 454 super(); 455 Object.assign(this, { 456 url, 457 reader: options.useXHR ? new XHRReader(url, options) : new FetchReader(url, options) 458 }); 459 } 460 461 set size(value) { 462 // ignored 463 } 464 465 get size() { 466 return this.reader.size; 467 } 468 469 async init() { 470 await this.reader.init(); 471 super.init(); 472 } 473 474 readUint8Array(index, length) { 475 return this.reader.readUint8Array(index, length); 476 } 477} 478 479class HttpRangeReader extends HttpReader { 480 481 constructor(url, options = {}) { 482 options.useRangeHeader = true; 483 super(url, options); 484 } 485} 486 487 488class Uint8ArrayReader extends Reader { 489 490 constructor(array) { 491 super(); 492 Object.assign(this, { 493 array, 494 size: array.length 495 }); 496 } 497 498 readUint8Array(index, length) { 499 return this.array.slice(index, index + length); 500 } 501} 502 503class Uint8ArrayWriter extends Writer { 504 505 init(initSize = 0) { 506 Object.assign(this, { 507 offset: 0, 508 array: new Uint8Array(initSize) 509 }); 510 super.init(); 511 } 512 513 writeUint8Array(array) { 514 const writer = this; 515 if (writer.offset + array.length > writer.array.length) { 516 const previousArray = writer.array; 517 writer.array = new Uint8Array(previousArray.length + array.length); 518 writer.array.set(previousArray); 519 } 520 writer.array.set(array, writer.offset); 521 writer.offset += array.length; 522 } 523 524 getData() { 525 return this.array; 526 } 527} 528 529class SplitDataReader extends Reader { 530 531 constructor(readers) { 532 super(); 533 this.readers = readers; 534 } 535 536 async init() { 537 const reader = this; 538 const { readers } = reader; 539 reader.lastDiskNumber = 0; 540 await Promise.all(readers.map(async diskReader => { 541 await diskReader.init(); 542 reader.size += diskReader.size; 543 })); 544 super.init(); 545 } 546 547 async readUint8Array(offset, length, diskNumber = 0) { 548 const reader = this; 549 const { readers } = this; 550 let result; 551 let currentDiskNumber = diskNumber; 552 if (currentDiskNumber == -1) { 553 currentDiskNumber = readers.length - 1; 554 } 555 let currentReaderOffset = offset; 556 while (currentReaderOffset >= readers[currentDiskNumber].size) { 557 currentReaderOffset -= readers[currentDiskNumber].size; 558 currentDiskNumber++; 559 } 560 const currentReader = readers[currentDiskNumber]; 561 const currentReaderSize = currentReader.size; 562 if (currentReaderOffset + length <= currentReaderSize) { 563 result = await readUint8Array(currentReader, currentReaderOffset, length); 564 } else { 565 const chunkLength = currentReaderSize - currentReaderOffset; 566 result = new Uint8Array(length); 567 result.set(await readUint8Array(currentReader, currentReaderOffset, chunkLength)); 568 result.set(await reader.readUint8Array(offset + chunkLength, length - chunkLength, diskNumber), chunkLength); 569 } 570 reader.lastDiskNumber = Math.max(currentDiskNumber, reader.lastDiskNumber); 571 return result; 572 } 573} 574 575class SplitDataWriter extends Stream { 576 577 constructor(writerGenerator, maxSize = 4294967295) { 578 super(); 579 const zipWriter = this; 580 Object.assign(zipWriter, { 581 diskNumber: 0, 582 diskOffset: 0, 583 size: 0, 584 maxSize, 585 availableSize: maxSize 586 }); 587 let diskSourceWriter, diskWritable, diskWriter; 588 const writable = new WritableStream({ 589 async write(chunk) { 590 const { availableSize } = zipWriter; 591 if (!diskWriter) { 592 const { value, done } = await writerGenerator.next(); 593 if (done && !value) { 594 throw new Error(ERR_ITERATOR_COMPLETED_TOO_SOON); 595 } else { 596 diskSourceWriter = value; 597 diskSourceWriter.size = 0; 598 if (diskSourceWriter.maxSize) { 599 zipWriter.maxSize = diskSourceWriter.maxSize; 600 } 601 zipWriter.availableSize = zipWriter.maxSize; 602 await initStream(diskSourceWriter); 603 diskWritable = value.writable; 604 diskWriter = diskWritable.getWriter(); 605 } 606 await this.write(chunk); 607 } else if (chunk.length >= availableSize) { 608 await writeChunk(chunk.slice(0, availableSize)); 609 await closeDisk(); 610 zipWriter.diskOffset += diskSourceWriter.size; 611 zipWriter.diskNumber++; 612 diskWriter = null; 613 await this.write(chunk.slice(availableSize)); 614 } else { 615 await writeChunk(chunk); 616 } 617 }, 618 async close() { 619 await diskWriter.ready; 620 await closeDisk(); 621 } 622 }); 623 Object.defineProperty(zipWriter, PROPERTY_NAME_WRITABLE, { 624 get() { 625 return writable; 626 } 627 }); 628 629 async function writeChunk(chunk) {
630 const chunkLength = chunk.length; 631 if (chunkLength) { 632 await diskWriter.ready; 633 await diskWriter.write(chunk); 634 diskSourceWriter.size += chunkLength; 635 zipWriter.size += chunkLength; 636 zipWriter.availableSize -= chunkLength; 637 } 638 } 639 640 async function closeDisk() { 641 diskWritable.size = diskSourceWriter.size; 642 await diskWriter.close(); 643 } 644 } 645} 646 647function isHttpFamily(url) { 648 const { baseURL } = getConfiguration(); 649 const { protocol } = new URL(url, baseURL); 650 return protocol == "http:" || protocol == "https:"; 651} 652 653async function initStream(stream, initSize) { 654 if (stream.init && !stream.initialized) { 655 await stream.init(initSize); 656 } 657} 658 659function initReader(reader) { 660 if (Array.isArray(reader)) { 661 reader = new SplitDataReader(reader); 662 } 663 if (reader instanceof ReadableStream) { 664 reader = { 665 readable: reader 666 }; 667 } 668 return reader; 669} 670 671function initWriter(writer) { 672 if (writer.writable === UNDEFINED_VALUE && typeof writer.next == FUNCTION_TYPE) { 673 writer = new SplitDataWriter(writer); 674 } 675 if (writer instanceof WritableStream) { 676 writer = { 677 writable: writer 678 }; 679 } 680 const { writable } = writer; 681 if (writable.size === UNDEFINED_VALUE) { 682 writable.size = 0; 683 } 684 const splitZipFile = writer instanceof SplitDataWriter; 685 if (!splitZipFile) { 686 Object.assign(writer, { 687 diskNumber: 0, 688 diskOffset: 0, 689 availableSize: Infinity, 690 maxSize: Infinity 691 }); 692 } 693 return writer; 694} 695 696function readUint8Array(reader, offset, size, diskNumber) { 697 return reader.readUint8Array(offset, size, diskNumber); 698} 699 700const SplitZipReader = SplitDataReader; 701const SplitZipWriter = SplitDataWriter; 702 703export { 704 initStream, 705 initReader, 706 initWriter, 707 readUint8Array, 708 Reader, 709 Writer, 710 TextReader, 711 TextWriter, 712 Data64URIReader, 713 Data64URIWriter, 714 BlobReader, 715 BlobWriter, 716 Uint8ArrayReader, 717 Uint8ArrayWriter, 718 HttpReader, 719 HttpRangeReader, 720 SplitZipReader, 721 SplitZipWriter, 722 SplitDataReader, 723 SplitDataWriter, 724 ERR_HTTP_RANGE, 725 ERR_ITERATOR_COMPLETED_TOO_SOON 726};
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.