Skip to content

Instantly share code, notes, and snippets.

@ibqn

ibqn/browser-ndjson-stream-reader.js

Forked from ornicar/browser-ndjson-stream-reader.js
Created January 28, 2022 22:12
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save ibqn/a5c1323fa7553e4d996975d8a3b029f9 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save ibqn/a5c1323fa7553e4d996975d8a3b029f9 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. @ornicar ornicar revised this gist Sep 11, 2021. 2 changed files with 2 additions and 2 deletions.
    2 changes: 1 addition & 1 deletion browser-ndjson-stream-reader.js
    View file
    Original file line number Diff line number Diff line change
    @@ -23,7 +23,7 @@ const readStream = processLine => response => {

    const parts = buf.split(matcher);
    buf = parts.pop();
    for (const i of parts) processLine(JSON.parse(i));
    for (const i of parts.filter(p => p)) processLine(JSON.parse(i));
    return loop();
    }
    });
    2 changes: 1 addition & 1 deletion nodejs-ndjson-stream-reader.js
    View file
    Original file line number Diff line number Diff line change
    @@ -15,7 +15,7 @@ const readStream = processLine => response => {

    const parts = buf.split(matcher);
    buf = parts.pop();
    for (const i of parts) processLine(JSON.parse(i));
    for (const i of parts.filter(p => p)) processLine(JSON.parse(i));
    });
    response.body.on('end', () => {
    if (buf.length > 0) processLine(JSON.parse(buf));
  2. @ornicar ornicar revised this gist Sep 10, 2021. 3 changed files with 29 additions and 1 deletion.
    2 changes: 1 addition & 1 deletion ndjson-stream-reader.js → browser-ndjson-stream-reader.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,4 @@
    /*
    /* FOR THE BROWSER
    Utility function to read a ND-JSON HTTP stream.
    `processLine` is a function taking a JSON object. It will be called with each element of the stream.
    `response` is the result of a `fetch` request.
    26 changes: 26 additions & 0 deletions nodejs-ndjson-stream-reader.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,26 @@
    /* FOR NODEJS
    Utility function to read a ND-JSON HTTP stream.
    `processLine` is a function taking a JSON object. It will be called with each element of the stream.
    `response` is the result of a `fetch` request.
    See usage example in the next file.
    */
    const readStream = processLine => response => {
    const matcher = /\r?\n/;
    const decoder = new TextDecoder();
    let buf = '';
    return new Promise((resolve, fail) => {
    response.body.on('data', v => {
    const chunk = decoder.decode(v, { stream: true });
    buf += chunk;

    const parts = buf.split(matcher);
    buf = parts.pop();
    for (const i of parts) processLine(JSON.parse(i));
    });
    response.body.on('end', () => {
    if (buf.length > 0) processLine(JSON.parse(buf));
    resolve();
    });
    response.body.on('error', fail);
    });
    };
    2 changes: 2 additions & 0 deletions usage.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,5 @@
    /* From browser or nodejs alike */

    const stream = fetch('https://lichess.org/api/tv/feed');
    // or any other ND-JSON endpoint such as:
    // const stream = fetch('https://lichess.org/api/games/user/neio',{headers:{Accept:'application/x-ndjson'}});
  3. @ornicar ornicar revised this gist Sep 10, 2021. 1 changed file with 3 additions and 1 deletion.
    4 changes: 3 additions & 1 deletion usage.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,6 @@
    const stream = fetch('https://lichess.org/api/games/user/neio',{headers:{Accept:'application/x-ndjson'}});
    const stream = fetch('https://lichess.org/api/tv/feed');
    // or any other ND-JSON endpoint such as:
    // const stream = fetch('https://lichess.org/api/games/user/neio',{headers:{Accept:'application/x-ndjson'}});

    const onMessage = obj => console.log(obj);
    const onComplete = () => console.log('The stream has completed');
  4. @ornicar ornicar created this gist Aug 25, 2021.
    32 changes: 32 additions & 0 deletions ndjson-stream-reader.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,32 @@
    /*
    Utility function to read a ND-JSON HTTP stream.
    `processLine` is a function taking a JSON object. It will be called with each element of the stream.
    `response` is the result of a `fetch` request.
    See usage example in the next file.
    */

    const readStream = processLine => response => {
    const stream = response.body.getReader();
    const matcher = /\r?\n/;
    const decoder = new TextDecoder();
    let buf = '';

    const loop = () =>
    stream.read().then(({ done, value }) => {
    if (done) {
    if (buf.length > 0) processLine(JSON.parse(buf));
    } else {
    const chunk = decoder.decode(value, {
    stream: true
    });
    buf += chunk;

    const parts = buf.split(matcher);
    buf = parts.pop();
    for (const i of parts) processLine(JSON.parse(i));
    return loop();
    }
    });

    return loop();
    }
    8 changes: 8 additions & 0 deletions usage.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,8 @@
    const stream = fetch('https://lichess.org/api/games/user/neio',{headers:{Accept:'application/x-ndjson'}});

    const onMessage = obj => console.log(obj);
    const onComplete = () => console.log('The stream has completed');

    stream
    .then(readStream(onMessage))
    .then(onComplete);