Skip to content

Instantly share code, notes, and snippets.

@nestarz

nestarz/fetchNDJSON.js

Last active July 6, 2024 05:02
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save nestarz/1fa7ae93fb83f1eafb1b88c3a84f2e02 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save nestarz/1fa7ae93fb83f1eafb1b88c3a84f2e02 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. nestarz revised this gist Oct 20, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion index.js
    View file
    Original file line number Diff line number Diff line change
    @@ -5,7 +5,7 @@
    let progress = 0;

    fetchJSONLD(file).then(({ response, reader }) => {
    let length = response.headers.get("Content-Length");
    const length = response.headers.get("Content-Length");
    let received = 0;

    const onReadChunk = async chunk => {
  2. nestarz revised this gist Oct 20, 2019. 1 changed file with 10 additions and 2 deletions.
    12 changes: 10 additions & 2 deletions index.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,15 +1,23 @@
    (async () => {
    const { fetchNDJSON } = await import("/fetchNDJSON.js");

    const results = [];
    let progress = 0;

    fetchJSONLD(file).then(({ response, reader }) => {
    length = response.headers.get("Content-Length");
    received = 0;
    let length = response.headers.get("Content-Length");
    let received = 0;

    const onReadChunk = async chunk => {
    if (chunk.done) {

    return;
    }

    // await timeout(1000);
    received += chunk.value.length;
    results.push(chunk.value);
    progress = received / length;

    reader.read().then(onReadChunk);
    };
  3. nestarz revised this gist Oct 20, 2019. No changes.
  4. nestarz revised this gist Oct 20, 2019. 1 changed file with 16 additions and 14 deletions.
    30 changes: 16 additions & 14 deletions index.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,16 +1,18 @@
    const { fetchNDJSON } = await import("/fetchNDJSON.js");
    fetchJSONLD(file).then(({ response, reader }) => {
    length = response.headers.get("Content-Length");
    received = 0;
    const onReadChunk = async chunk => {
    if (chunk.done) {
    return;
    }
    // await timeout(1000);
    received += chunk.value.length;
    results.push(chunk.value);
    (async () => {
    const { fetchNDJSON } = await import("/fetchNDJSON.js");
    fetchJSONLD(file).then(({ response, reader }) => {
    length = response.headers.get("Content-Length");
    received = 0;
    const onReadChunk = async chunk => {
    if (chunk.done) {
    return;
    }
    // await timeout(1000);
    received += chunk.value.length;
    results.push(chunk.value);

    reader.read().then(onReadChunk);
    };
    reader.read().then(onReadChunk);
    };
    reader.read().then(onReadChunk);
    });
    });
    })();
  5. nestarz revised this gist Oct 20, 2019. 2 changed files with 1 addition and 1 deletion.
    0 fetchJSONLD.js → fetchNDJSON.js
    View file
    File renamed without changes.
    2 changes: 1 addition & 1 deletion index.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,4 @@
    const { fetchJSONLD } = await import("/fetchJSONLD.js");
    const { fetchNDJSON } = await import("/fetchNDJSON.js");
    fetchJSONLD(file).then(({ response, reader }) => {
    length = response.headers.get("Content-Length");
    received = 0;
  6. nestarz revised this gist Oct 20, 2019. 1 changed file with 1 addition and 0 deletions.
    1 change: 1 addition & 0 deletions fetchJSONLD.js
    View file
    Original file line number Diff line number Diff line change
    @@ -25,6 +25,7 @@ const fetchJSONLD = url =>
    response,
    reader: response.body
    .pipeThrough(new TextDecoderStream())
    // Needed to stream by line and then JSON parse the line
    .pipeThrough(splitStream("\n"))
    .pipeThrough(parseJSON())
    .getReader()
  7. nestarz revised this gist Oct 20, 2019. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion index.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,4 @@
    const { fetchJSONLD } = await import("/stream-jsonld.js");
    const { fetchJSONLD } = await import("/fetchJSONLD.js");
    fetchJSONLD(file).then(({ response, reader }) => {
    length = response.headers.get("Content-Length");
    received = 0;
  8. nestarz created this gist Oct 20, 2019.
    33 changes: 33 additions & 0 deletions fetchJSONLD.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,33 @@
    const parseJSON = () =>
    new TransformStream({
    transform(chunk, controller) {
    controller.enqueue(JSON.parse(chunk));
    }
    });

    const splitStream = splitOn => {
    let buffer = "";
    return new TransformStream({
    transform(chunk, controller) {
    buffer += chunk;
    const parts = buffer.split(splitOn);
    parts.slice(0, -1).forEach(part => controller.enqueue(part));
    buffer = parts[parts.length - 1];
    },
    flush(controller) {
    if (buffer) controller.enqueue(buffer);
    }
    });
    };

    const fetchJSONLD = url =>
    fetch(url).then(response => ({
    response,
    reader: response.body
    .pipeThrough(new TextDecoderStream())
    .pipeThrough(splitStream("\n"))
    .pipeThrough(parseJSON())
    .getReader()
    }));

    export { fetchJSONLD };
    16 changes: 16 additions & 0 deletions index.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,16 @@
    const { fetchJSONLD } = await import("/stream-jsonld.js");
    fetchJSONLD(file).then(({ response, reader }) => {
    length = response.headers.get("Content-Length");
    received = 0;
    const onReadChunk = async chunk => {
    if (chunk.done) {
    return;
    }
    // await timeout(1000);
    received += chunk.value.length;
    results.push(chunk.value);

    reader.read().then(onReadChunk);
    };
    reader.read().then(onReadChunk);
    });