import { Chunk } from '@livestore/utils/effect'
const textEncoder = new TextEncoder()
/**
* Configuration describing how to break a chunk into smaller payload-safe chunks.
*/
export interface ChunkingOptions {
/** Maximum number of items that may appear in any emitted chunk. */
readonly maxItems: number
/** Maximum encoded byte size allowed for any emitted chunk. */
readonly maxBytes: number
/**
* Callback that produces a JSON-serialisable structure whose byte size should
* fit within {@link maxBytes}. This lets callers control framing overhead.
*/
readonly encode: (items: ReadonlyArray) => unknown
}
/**
* Derives a function that splits an input chunk into sub-chunks confined by
* both item count and encoded byte size limits. Designed for transports with
* strict frame caps (e.g. Cloudflare hibernated WebSockets).
*/
export const splitChunkBySize =
(options: ChunkingOptions) =>
(chunk: Chunk.Chunk): Chunk.Chunk> => {
const maxItems = Math.max(1, options.maxItems)
const maxBytes = Math.max(1, options.maxBytes)
const encode = options.encode
const measure = (items: ReadonlyArray) => {
const encoded = encode(items)
return textEncoder.encode(JSON.stringify(encoded)).byteLength
}
const items = Chunk.toReadonlyArray(chunk)
if (items.length === 0) {
return Chunk.fromIterable>([])
}
const result: Array> = []
let current: Array = []
const flushCurrent = () => {
if (current.length > 0) {
result.push(Chunk.fromIterable(current))
current = []
}
}
for (const item of items) {
current.push(item)
const exceedsLimit = current.length > maxItems || measure(current) > maxBytes
if (exceedsLimit) {
// remove the item we just added and emit the previous chunk if it exists
const last = current.pop()!
flushCurrent()
if (last !== undefined) {
current = [last]
const singleItemTooLarge = measure(current) > maxBytes
if (singleItemTooLarge || current.length > maxItems) {
// Emit the oversized item on its own; downstream can decide how to handle it.
result.push(Chunk.of(last))
current = []
}
}
}
}
flushCurrent()
return Chunk.fromIterable(result)
}