Skip to content

Instantly share code, notes, and snippets.

@phipex

phipex/pg-cursor iterator example.js

Forked from zachsa/pg-cursor iterator example.js
Last active June 23, 2021 20:42
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save phipex/49c866032ee46a0c7c4dd896984b12f5 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save phipex/49c866032ee46a0c7c4dd896984b12f5 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. phipex revised this gist Jun 23, 2021. 1 changed file with 3 additions and 4 deletions.
    7 changes: 3 additions & 4 deletions pg-cursor iterator example.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,8 +1,7 @@
    const createIterator = async sql => {
    const createIterator = async (sql, batchSize = 100) => {
    const client = await pool.connect()
    const cursor = client.query(new Cursor(sql))
    const batchSize = 100


    return (async function getRows(client, cursor, batchSize) {
    let done = false

    @@ -30,7 +29,7 @@ const createIterator = async sql => {
    }

    // Used like so:
    let iterator = await createIterator ('select ... from ...')
    let iterator = await createIterator ('select ... from ...', 200)
    while (!iterator.done) {
    let rows = iterator.rows
    // do stuff with rows
  2. @zachsa zachsa revised this gist Sep 25, 2019. 1 changed file with 32 additions and 38 deletions.
    70 changes: 32 additions & 38 deletions pg-cursor iterator example.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,44 +1,38 @@
    const rowIterator = async (client, cursor, batchSize) => {
    let done = false
    const createIterator = async sql => {
    const client = await pool.connect()
    const cursor = client.query(new Cursor(sql))
    const batchSize = 100

    return (async function getRows(client, cursor, batchSize) {
    let done = false

    // Get next rows
    const rows = await new Promise((resolve, reject) =>
    cursor.read(batchSize, (err, rows) => {
    if (err) reject(err)
    else resolve(rows)
    })
    )
    // Get next rows
    const rows = await new Promise((resolve, reject) =>
    cursor.read(
    batchSize,
    (err, rows) => err ? reject(err) : resolve(rows)
    )
    )

    // Check if iteration is finished
    if (rows.length < 1) {
    done = true
    cursor.close(async () => {
    await client.release()
    })
    }
    // Check if iteration is finished
    if (rows.length < 1) {
    done = true
    cursor.close(() => client.release())
    }

    // Return the iterator
    return {
    done,
    rows,
    next: async () => await rowIterator(client, cursor, batchSize)
    }
    // Return the iterator
    return {
    done,
    rows,
    next: async () => await getRows(client, cursor, batchSize)
    }
    })(client, cursor, batchSize)
    }

    /**
    * Use in other files like this:
    * const pgCursor = iPgQuery("select * from ...")
    * const rowIterator = pgCursor.createRowIterator(1000)
    * while (!rowIterator.done) {
    * let rows = rowIterator.rows
    * // do stuff with rows
    * rowIterator = await rowIterator.next()
    * }
    */
    export const iPgQuery = async sql => {
    const client = await pool.connect()
    const cursor = client.query(new Cursor(sql))
    return {
    createRowIterator: batchSize => rowIterator(client, cursor, batchSize)
    }
    // Used like so:
    let iterator = await createIterator ('select ... from ...')
    while (!iterator.done) {
    let rows = iterator.rows
    // do stuff with rows
    iterator = await rowIterator.next()
    }
  3. @zachsa zachsa created this gist Sep 9, 2019.
    44 changes: 44 additions & 0 deletions pg-cursor iterator example.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,44 @@
    const rowIterator = async (client, cursor, batchSize) => {
    let done = false

    // Get next rows
    const rows = await new Promise((resolve, reject) =>
    cursor.read(batchSize, (err, rows) => {
    if (err) reject(err)
    else resolve(rows)
    })
    )

    // Check if iteration is finished
    if (rows.length < 1) {
    done = true
    cursor.close(async () => {
    await client.release()
    })
    }

    // Return the iterator
    return {
    done,
    rows,
    next: async () => await rowIterator(client, cursor, batchSize)
    }
    }

    /**
    * Use in other files like this:
    * const pgCursor = iPgQuery("select * from ...")
    * const rowIterator = pgCursor.createRowIterator(1000)
    * while (!rowIterator.done) {
    * let rows = rowIterator.rows
    * // do stuff with rows
    * rowIterator = await rowIterator.next()
    * }
    */
    export const iPgQuery = async sql => {
    const client = await pool.connect()
    const cursor = client.query(new Cursor(sql))
    return {
    createRowIterator: batchSize => rowIterator(client, cursor, batchSize)
    }
    }