Skip to content

Instantly share code, notes, and snippets.

@developit

developit/run

Last active June 5, 2024 13:34
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save developit/4cba2e9ec7d1e09b8245febe7bad82bd to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save developit/4cba2e9ec7d1e09b8245febe7bad82bd to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. developit revised this gist Jun 5, 2024. 1 changed file with 10 additions and 70 deletions.
    80 changes: 10 additions & 70 deletions run
    View file
    Original file line number Diff line number Diff line change
    @@ -1,6 +1,6 @@
    #!/usr/bin/env node

    /* eslint-disable no-param-reassign, no-fallthrough, guard-for-in */
    /* eslint-disable no-param-reassign, no-fallthrough, guard-for-in, require-atomic-updates */
    const path = require('node:path');
    const {Worker} = require('node:worker_threads');

    @@ -261,8 +261,6 @@ parentPort.once('message', ([mod, filename, args, env, compiledCode]) => {
    module.id = basename(filename);
    module.filename = __filename = filename;
    module.path = __dirname = dirname(__filename);
    // let p = '';
    // module.paths = module.path.split(sep).map((seg) => (p += seg + sep) + 'node_modules').reverse();
    Object.assign(process.env, env);
    parentPort.postMessage(0);
    const exit = process.exit.bind(process);
    @@ -282,7 +280,6 @@ const workerPool = {
    create: () => new Worker(workerShim, {eval: true}),
    };
    workerPool.workers.push(workerPool.create());
    // if (process.env.POOL) workerPool.workers.push(workerPool.create());

    const pkgCache = new Map();
    function getPkg(filename) {
    @@ -313,72 +310,14 @@ async function runNode(script, args, env, compiledCode) {
    filename.endsWith('.mjs') ||
    ((await getPkg(filename)).type === 'module' && !filename.endsWith('.cjs'));

    return new Promise((resolve, reject) => {
    return new Promise((resolve) => {
    const end = done.submeasure('worker start');

    /*
    let worker;
    if (process.env.POOL) {
    worker = workerPool.get();
    worker.postMessage([isModule, filename, args, $env(env), compiledCode]);
    } else {
    const scriptStr = JSON.stringify(filename);
    const shim = `{
    console.log(process.argv.slice());
    process.argv[1] = ${scriptStr};
    ${
    // eslint-disable-next-line no-nested-ternary
    compiledCode
    ? `require('vm').runInThisContext(${JSON.stringify(
    compiledCode,
    )}, {filename: ${scriptStr}});`
    : isModule
    ? `import(${scriptStr});`
    : `require('vm').runInThisContext(require('fs').readFileSync(${scriptStr}, 'utf-8'), {filename: ${scriptStr}});`
    }
    }`;
    // const shim =
    // compiledCode || (isModule && `import(${JSON.stringify(filename)});`);
    // worker = new Worker(shim || filename, {
    worker = new Worker(shim, {
    env: $env(env),
    argv: args,
    eval: true,
    // eval: Boolean(shim),
    // resourceLimits: {
    // maxOldGenerationSizeMb: 9000,
    // maxYoungGenerationSizeMb: 9000,
    // },
    });
    }
    */
    const worker = workerPool.get();
    worker.postMessage([isModule, filename, args, $env(env), compiledCode]);
    end();
    worker.once('message', done.submeasure('worker init'));
    // worker.once('online', done.submeasure('worker init'));
    // worker.on('error', (err) => {
    // let stack = err.stack;
    // try {
    // const ref = err.stack?.match(/^\s*at .*?\((\/[^)]*?):(\d+):(\d+)\)?$/m);
    // if (ref) {
    // const [, file, lineNo, colNo] = ref;
    // const code = require('node:fs').readFileSync(file, 'utf-8');
    // const loc = `${file}:${lineNo}:${colNo}`;
    // const line = code.split('\n')[lineNo - 1];
    // stack = `\n on ${loc}:\n ${line}\n ${'-'.repeat(colNo | 0)}^`;
    // }
    // } catch (_) {
    // // ignore, use native stack
    // }
    // console.error(red(`${bold('Error')}: ${err}${dim(stack)}`));
    // done();
    // reject(err);
    // });
    worker.once('exit', (exitCode) => {
    done();
    resolve(exitCode);
    });
    worker.once('exit', done);
    worker.once('exit', resolve);
    end();
    });
    }

    @@ -503,12 +442,13 @@ async function runScript(scriptName, args, env) {
    }

    (async () => {
    const script = process.argv[2];
    if (!script) panic('No script name provided.');
    const start = performance.now();
    const cmd = path.basename(process.argv[1]);
    const runner = cmd === 'exec' ? runBin : runScript;
    const name = process.argv[2];
    if (!name) panic('No script name provided.');
    try {
    // eslint-disable-next-line require-atomic-updates
    process.exitCode = await runScript(script, process.argv.slice(3));
    process.exitCode = await runner(name, process.argv.slice(3));
    } finally {
    let worker;
    while ((worker = workerPool.workers?.pop())) worker.terminate();
  2. developit revised this gist Jun 5, 2024. 1 changed file with 153 additions and 41 deletions.
    194 changes: 153 additions & 41 deletions run
    View file
    Original file line number Diff line number Diff line change
    @@ -1,14 +1,5 @@
    #!/usr/bin/env node

    /**
    * run: run npm scripts faster and with fewer resources than should be possible.
    * Parses and executes in-process instead of spawning nested node+sh processes. Why:
    * - npm scripts that run other npm scripts have no overhead
    * - npm scripts that use ts-node get prebundled and cached by esbuild instead of using ts-node
    * Usage:
    * ./run my-script-name --a=b
    */

    /* eslint-disable no-param-reassign, no-fallthrough, guard-for-in */
    const path = require('node:path');
    const {Worker} = require('node:worker_threads');
    @@ -22,6 +13,11 @@ if (debugEnv) {
    }
    }
    const readFile = (file) => require('node:fs/promises').readFile(file, 'utf-8');
    const exists = (filename) =>
    require('node:fs/promises')
    .access(filename)
    .then(() => true)
    .catch(() => false);

    const $env = (env) => (env ? {...process.env, ...env} : process.env);

    @@ -98,6 +94,12 @@ function parseShell(script) {
    context = ARGS;
    }
    } else if (context === ARGS) {
    // unquote
    if (buf[0] === "'" && buf.at(-1) === "'") {
    buf = buf.slice(1, -1);
    } else if (buf[0] === '"' && buf.at(-1) === '"') {
    buf = buf.slice(1, -1).replaceAll('\\"', '"');
    }
    command.args.push(buf);
    }
    buf = '';
    @@ -134,6 +136,10 @@ function parseShell(script) {
    }
    break;
    case 'if':
    if (context !== INITIAL) {
    buf += op;
    continue;
    }
    logic.push({
    type: 'conditional',
    test: [],
    @@ -145,16 +151,28 @@ function parseShell(script) {
    context = INITIAL;
    break;
    case 'then':
    if (context !== INITIAL) {
    buf += op;
    continue;
    }
    seq = logic.at(-1)?.consequent;
    command = null;
    context = INITIAL;
    break;
    case 'else':
    if (context !== INITIAL) {
    buf += op;
    continue;
    }
    seq = logic.at(-1)?.alternate;
    command = null;
    context = INITIAL;
    break;
    case 'fi':
    if (context !== INITIAL) {
    buf += op;
    continue;
    }
    seq = root;
    if (logic.length) seq.push(logic.pop());
    break;
    @@ -235,21 +253,36 @@ async function run(seq, env = {}) {
    }

    const workerShim = `
    require('worker_threads').parentPort.once('message', ([mod, filename, args, env, compiledCode]) => {
    process.argv.splice(2, process.argv.length, ...args);
    const {parentPort} = require('node:worker_threads');
    const {dirname, basename, sep} = require('path');
    parentPort.once('message', ([mod, filename, args, env, compiledCode]) => {
    process.stdout.isTTY = process.stdin.isTTY = process.stderr.isTTY = true;
    process.argv = [process.argv[0], filename].concat(args);
    module.id = basename(filename);
    module.filename = __filename = filename;
    module.path = __dirname = dirname(__filename);
    // let p = '';
    // module.paths = module.path.split(sep).map((seg) => (p += seg + sep) + 'node_modules').reverse();
    Object.assign(process.env, env);
    require('worker_threads').parentPort.postMessage('done');
    parentPort.postMessage(0);
    const exit = process.exit.bind(process);
    process.exit = (exitCode) => {
    process.stdout.end();
    process.setUncaughtExceptionCaptureCallback(() => {});
    setTimeout(exit, 0, exitCode);
    };
    if (mod && !compiledCode) return import(filename);
    const code = compiledCode || require('fs').readFileSync(filename, 'utf-8');
    require('vm').runInThisContext(code, {filename});
    require = require('node:module').createRequire(require.resolve(filename));
    require('vm').runInThisContext(compiledCode || require('fs').readFileSync(filename, 'utf-8'), {filename});
    });
    `;
    const workerPool = {
    workers: [],
    get: () => workerPool.workers.pop() ?? workerPool.create(),
    create: () => new Worker(workerShim, {eval: true}),
    };
    if (!process.env.INLINE) workerPool.workers.push(workerPool.create());
    workerPool.workers.push(workerPool.create());
    // if (process.env.POOL) workerPool.workers.push(workerPool.create());

    const pkgCache = new Map();
    function getPkg(filename) {
    @@ -265,6 +298,13 @@ function getPkg(filename) {
    }

    async function runNode(script, args, env, compiledCode) {
    if (
    process.env.ESBUILD === 'force' &&
    compiledCode === undefined &&
    !script.endsWith('.built.mjs')
    ) {
    return runTs(script, args, env);
    }
    const scriptId = `node(${script} ${args.join(' ')})`;
    const done = measure(scriptId);
    const filename = path.resolve(script);
    @@ -275,28 +315,66 @@ async function runNode(script, args, env, compiledCode) {

    return new Promise((resolve, reject) => {
    const end = done.submeasure('worker start');

    /*
    let worker;
    if (process.env.POOL) {
    worker = workerPool.get();
    worker.postMessage([isModule, filename, args, $env(env), compiledCode]);
    } else {
    const scriptStr = JSON.stringify(filename);
    const shim = `{
    console.log(process.argv.slice());
    process.argv[1] = ${scriptStr};
    ${
    // eslint-disable-next-line no-nested-ternary
    compiledCode
    ? `require('vm').runInThisContext(${JSON.stringify(
    compiledCode,
    )}, {filename: ${scriptStr}});`
    : isModule
    ? `import(${scriptStr});`
    : `require('vm').runInThisContext(require('fs').readFileSync(${scriptStr}, 'utf-8'), {filename: ${scriptStr}});`
    }
    }`;
    // const shim =
    // compiledCode || (isModule && `import(${JSON.stringify(filename)});`);
    // worker = new Worker(shim || filename, {
    worker = new Worker(shim, {
    env: $env(env),
    argv: args,
    eval: true,
    // eval: Boolean(shim),
    // resourceLimits: {
    // maxOldGenerationSizeMb: 9000,
    // maxYoungGenerationSizeMb: 9000,
    // },
    });
    }
    */
    const worker = workerPool.get();
    worker.postMessage([isModule, filename, args, $env(env), compiledCode]);
    end();
    worker.once('message', done.submeasure('worker init'));
    worker.on('error', (err) => {
    let stack = err.stack;
    try {
    const ref = err.stack?.match(/^\s*at .*?\((\/[^)]*?):(\d+):(\d+)\)?$/m);
    if (ref) {
    const [, file, lineNo, colNo] = ref;
    const code = require('node:fs').readFileSync(file, 'utf-8');
    const loc = `${file}:${lineNo}:${colNo}`;
    const line = code.split('\n')[lineNo - 1];
    stack = `\n on ${loc}:\n ${line}\n ${'-'.repeat(colNo | 0)}^`;
    }
    } catch (_) {
    // ignore, use native stack
    }
    console.error(red(`${bold('Error')}: ${err}${dim(stack)}`));
    done();
    reject(err);
    });
    // worker.once('online', done.submeasure('worker init'));
    // worker.on('error', (err) => {
    // let stack = err.stack;
    // try {
    // const ref = err.stack?.match(/^\s*at .*?\((\/[^)]*?):(\d+):(\d+)\)?$/m);
    // if (ref) {
    // const [, file, lineNo, colNo] = ref;
    // const code = require('node:fs').readFileSync(file, 'utf-8');
    // const loc = `${file}:${lineNo}:${colNo}`;
    // const line = code.split('\n')[lineNo - 1];
    // stack = `\n on ${loc}:\n ${line}\n ${'-'.repeat(colNo | 0)}^`;
    // }
    // } catch (_) {
    // // ignore, use native stack
    // }
    // console.error(red(`${bold('Error')}: ${err}${dim(stack)}`));
    // done();
    // reject(err);
    // });
    worker.once('exit', (exitCode) => {
    done();
    resolve(exitCode);
    @@ -307,20 +385,43 @@ async function runNode(script, args, env, compiledCode) {
    async function runTs(script, args, env) {
    const end = measure(`esbuild(${script})`);
    const {build} = require('esbuild');
    const outfile = script.endsWith('.mjs')
    ? script.replace(/([^/]+)\.mjs$/, '.$1.built.mjs')
    : undefined;
    if (!outfile && !/\.[a-z]+$/.test(script)) {
    const [ext, index] = await Promise.all([
    exists(`${script}.ts`),
    exists(`${script}/index.ts`),
    ]);
    if (ext) script += '.ts';
    else if (index) script += '/index.ts';
    }
    const result = await build({
    bundle: true,
    entryPoints: [script],
    target: 'node20',
    platform: 'node',
    format: 'cjs',
    format: outfile ? 'esm' : 'cjs',
    packages: 'external',
    write: false,
    write: Boolean(outfile),
    outfile,
    treeShaking: true,
    minifyIdentifiers: true,
    minifySyntax: true,
    nodePaths: [],
    });
    if (result.errors.length) {
    panic(`TS build of ${script} failed:\n${result.errors.join('\n')}`);
    }

    if (outfile) {
    end();
    try {
    return await runNode(outfile, args, env);
    } finally {
    require('node:fs').unlink(outfile, Object);
    }
    }
    let code = result.outputFiles[0].text;

    // move top-level requires to callsites so they only get evaluated when surrounding code is executed
    @@ -354,12 +455,14 @@ async function runBin(bin, args, env) {
    if (Object.prototype.hasOwnProperty.call(BIN, bin)) {
    return BIN[bin](args, env);
    }
    let resolvedBin = bin;
    try {
    const done = measure(`resolve-bin(${bin} ${args.join(' ')})`);
    const code = await readFile(`node_modules/.bin/${bin}`);
    resolvedBin = `node_modules/.bin/${bin}`;
    // exec node "$basedir/../vite/bin/vite.js" "$@"
    const match = code.match(/exec node +"([^"]+)" "\$@"/)[1];
    if (!match) panic(`Unknown format at node_modules/.bin/${bin}`);
    const match = code.match(/exec +node +"([^"]+)" +"\$@"/)?.[1];
    if (!match) throw Error(`Unknown format at node_modules/.bin/${bin}`);
    const binFile = path.resolve(
    match.replace('$basedir', './node_modules/.bin'),
    );
    @@ -368,19 +471,28 @@ async function runBin(bin, args, env) {
    return runNode(binRel, args, env);
    } catch (err) {
    const done = measure(`run-bin(${bin} ${args.join(' ')})`);
    console.error(`failed to run "${bin}" seamlessly:`, err);
    // console.error(`failed to run "${bin}" seamlessly:`, err);
    return new Promise((resolve) => {
    const {spawn} = require('node:child_process');
    const proc = spawn(bin, args, {env: $env(env), stdio: 'inherit'});
    const proc = spawn(resolvedBin, args, {env: $env(env), stdio: 'inherit'});
    // proc.on('error', reject);
    proc.once('exit', (exitCode) => done(resolve(exitCode)));
    proc.once('exit', (exitCode) => {
    done();
    resolve(exitCode);
    });
    });
    }
    }

    async function runScript(scriptName, args, env) {
    const script = (await pkgPromise).scripts[scriptName];
    if (!script) panic(`Unknown script "${scriptName}".`);
    if (!script) {
    const bin = path.basename(scriptName);
    if (await exists(`node_modules/.bin/${bin}`)) {
    return runBin(bin, args, env);
    }
    panic(`Unknown script "${scriptName}".`);
    }
    let scriptWithArgs = script;
    for (const arg of args) scriptWithArgs += ` ${arg}`;
    const done = measure('parse');
  3. developit revised this gist May 31, 2024. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion run
    View file
    Original file line number Diff line number Diff line change
    @@ -4,7 +4,7 @@
    * run: run npm scripts faster and with fewer resources than should be possible.
    * Parses and executes in-process instead of spawning nested node+sh processes. Why:
    * - npm scripts that run other npm scripts have no overhead
    * - npm scripts that use ts-node get prebundled and cached by esbuild
    * - npm scripts that use ts-node get prebundled and cached by esbuild instead of using ts-node
    * Usage:
    * ./run my-script-name --a=b
    */
  4. developit revised this gist May 31, 2024. 1 changed file with 1 addition and 33 deletions.
    34 changes: 1 addition & 33 deletions run
    View file
    Original file line number Diff line number Diff line change
    @@ -275,35 +275,6 @@ async function runNode(script, args, env, compiledCode) {

    return new Promise((resolve, reject) => {
    const end = done.submeasure('worker start');

    // if (process.env.POOL) {
    // worker = workerPool.get();
    // worker.postMessage([isModule, filename, args, $env(env), compiledCode]);
    // } else {
    // const scriptStr = JSON.stringify(filename);
    // const shim = `{
    // process.argv[1] = ${scriptStr};
    // ${
    // compiledCode
    // ? `require('vm').runInThisContext(${JSON.stringify(
    // compiledCode,
    // )}, {filename: ${scriptStr}});`
    // : isModule
    // ? `import(${scriptStr});`
    // : `require('vm').runInThisContext(require('fs').readFileSync(${scriptStr}, 'utf-8'), {filename: ${scriptStr}});`
    // }
    // require('worker_threads').parentPort.postMessage(0);
    // }`;
    // worker = new Worker(shim, {
    // env: $env(env),
    // argv: args,
    // eval: true,
    // // resourceLimits: {
    // // maxOldGenerationSizeMb: 9000,
    // // maxYoungGenerationSizeMb: 9000,
    // // },
    // });
    // }
    const worker = workerPool.get();
    worker.postMessage([isModule, filename, args, $env(env), compiledCode]);
    end();
    @@ -402,10 +373,7 @@ async function runBin(bin, args, env) {
    const {spawn} = require('node:child_process');
    const proc = spawn(bin, args, {env: $env(env), stdio: 'inherit'});
    // proc.on('error', reject);
    proc.once('exit', (exitCode) => {
    done();
    resolve(exitCode);
    });
    proc.once('exit', (exitCode) => done(resolve(exitCode)));
    });
    }
    }
  5. developit revised this gist May 31, 2024. 1 changed file with 3 additions and 1 deletion.
    4 changes: 3 additions & 1 deletion run
    View file
    Original file line number Diff line number Diff line change
    @@ -2,7 +2,9 @@

    /**
    * run: run npm scripts faster and with fewer resources than should be possible.
    * Parses and executes in-process instead of spawning nested node+sh processes.
    * Parses and executes in-process instead of spawning nested node+sh processes. Why:
    * - npm scripts that run other npm scripts have no overhead
    * - npm scripts that use ts-node get prebundled and cached by esbuild
    * Usage:
    * ./run my-script-name --a=b
    */
  6. developit revised this gist May 31, 2024. 2 changed files with 24 additions and 570 deletions.
    392 changes: 24 additions & 368 deletions run
    View file
    Original file line number Diff line number Diff line change
    @@ -1,9 +1,15 @@
    #!/usr/bin/env node

    /* eslint-disable no-param-reassign, no-fallthrough, guard-for-in, no-nested-ternary */

    const {Worker} = require('node:worker_threads');
    /**
    * run: run npm scripts faster and with fewer resources than should be possible.
    * Parses and executes in-process instead of spawning nested node+sh processes.
    * Usage:
    * ./run my-script-name --a=b
    */

    /* eslint-disable no-param-reassign, no-fallthrough, guard-for-in */
    const path = require('node:path');
    const {Worker} = require('node:worker_threads');

    const DEBUG = {parsing: false, timing: false};
    const debugEnv = process.env.DEBUG;
    @@ -15,10 +21,6 @@ if (debugEnv) {
    }
    const readFile = (file) => require('node:fs/promises').readFile(file, 'utf-8');

    // process.env.NODE_OPTIONS = `--no-addons --no-deprecation --no-force-async-hooks-checks --no-global-search-paths ${
    // process.env.NODE_OPTIONS || ''
    // }`;

    const $env = (env) => (env ? {...process.env, ...env} : process.env);

    let markId = 0;
    @@ -65,26 +67,24 @@ const COMMAND = 1;
    const ARGS = 2;

    function parseShell(script) {
    const tokenizer = /(if|then|fi|&&|\|\||[ ;"'`()=])/g;
    script += ' ';
    let token;
    let index = 0;
    const tokenizer = /(if|then|fi|&&|\|\||[ ;"'`()=])/g;
    const expectedOps = {
    ')': '(',
    fi: 'if',
    };
    const stack = [];
    const logic = [];
    let lastOp;
    let context = INITIAL;
    let token;
    let index = 0;
    let lastOp;
    let lastEnv;
    let buf = '';
    let command = {name: '', args: [], env: {}};
    const root = [];
    let seq = root;
    const expectedOps = {
    ')': '(',
    // then: 'if',
    fi: 'if',
    };
    const commit = () => {
    // console.log({context, buf, lastEnv});
    if (context === INITIAL) {
    if (lastEnv) {
    if (!command) seq.push((command = {name: '', args: [], env: {}}));
    @@ -102,7 +102,6 @@ function parseShell(script) {
    };
    while ((token = tokenizer.exec(script))) {
    let op = token[0];
    // console.log(token);
    buf += script.substring(index, token.index);
    index = tokenizer.lastIndex;
    switch (op) {
    @@ -111,14 +110,12 @@ function parseShell(script) {
    if (lastOp) continue;
    seq.push({type: op});
    case ';':
    // if (lastOp === 'if') continue;
    if (lastOp) continue;
    commit();
    command = null;
    context = INITIAL;
    break;
    case ' ':
    // console.log('space', {context, buf, lastOp, lastEnv});
    if (!buf) continue;
    if (lastOp) {
    if (buf) buf += ' ';
    @@ -142,19 +139,16 @@ function parseShell(script) {
    alternate: [],
    });
    seq = logic.at(-1)?.test;
    // seq.push((command = {name: '', args: [], env: {}}));
    command = null;
    context = INITIAL;
    break;
    case 'then':
    seq = logic.at(-1)?.consequent;
    // seq.push((command = {name: '', args: [], env: {}}));
    command = null;
    context = INITIAL;
    break;
    case 'else':
    seq = logic.at(-1)?.alternate;
    // seq.push((command = {name: '', args: [], env: {}}));
    command = null;
    context = INITIAL;
    break;
    @@ -169,15 +163,9 @@ function parseShell(script) {
    case '`':
    buf += op;
    default:
    // if (op === 'then') {
    // seq.push({type: 'if'});
    // seq.push((command = {name: '', args: [], env: {}}));
    // continue;
    // }
    if (op in expectedOps) {
    const start = expectedOps[op];
    if (lastOp !== start) {
    // console.log({context, stack, buf, op, start, lastOp, seq});
    const line = `${script}\n${'-'.repeat(token.index)}^`;
    panic(`unexpected "${op}"\n${dim(line)}`);
    }
    @@ -189,7 +177,6 @@ function parseShell(script) {
    } else {
    stack.push(op);
    lastOp = op;
    // continue;
    }
    break;
    }
    @@ -211,10 +198,8 @@ async function run(seq, env = {}) {
    case 'conditional':
    if (await run(node.test, env)) {
    prev = await run(node.consequent, env);
    // await run(node.consequent, env);
    } else if (node.alternate) {
    prev = await run(node.alternate, env);
    // await run(node.alternate, env);
    }
    break;
    default:
    @@ -277,10 +262,7 @@ function getPkg(filename) {
    return prom;
    }

    async function runNode(script, args, env, compiledCode, hash) {
    // if (process.env.FORCE_ESBUILD && compiledCode === undefined) {
    // return runTs(script, args, env);
    // }
    async function runNode(script, args, env, compiledCode) {
    const scriptId = `node(${script} ${args.join(' ')})`;
    const done = measure(scriptId);
    const filename = path.resolve(script);
    @@ -290,124 +272,9 @@ async function runNode(script, args, env, compiledCode, hash) {
    ((await getPkg(filename)).type === 'module' && !filename.endsWith('.cjs'));

    return new Promise((resolve, reject) => {
    // if (process.env.FORK) {
    // const {fork} = require('node:child_process');
    // return fork(script, args, {env, stdio: 'inherit'})
    // .once('error', reject)
    // .once('exit', resolve);
    // }

    const end = done.submeasure('worker start');
    // let worker;

    if (process.env.INLINE) {
    /** @type {typeof process} */
    const processShim = {
    ...process,
    argv: [process.argv[0], filename, ...args],
    exitCode: 0,
    exit(code) {
    processShim.exitCode = code;
    done();
    resolve(processShim.exitCode);
    },
    };

    const _import = async (id) => {
    const oldArgv = process.argv;
    const oldExit = process.exit;
    process.argv = processShim.argv;
    process.exit = processShim.exit;
    try {
    return await import(id);
    } finally {
    process.argv = oldArgv;
    process.exit = oldExit;
    }
    };

    const _require = (id) => {
    if (id === 'process' || id === 'node:process') return processShim;
    return require(id);
    };

    // if (process.env.LAZYREQUIRE) {
    // const {builtinModules} = require('node:module');
    // const requireCache = new Map();
    // const dlv = (obj, keyPath) => {
    // for (const key of keyPath) obj = obj?.[key];
    // return obj;
    // };
    // const _export = (root, keyPath) =>
    // dlv(root, keyPath) ?? dlv(root.default, keyPath);
    // const recursiveProxy = (load, loaded, keyPath = []) =>
    // new Proxy(function () {}, {
    // get(target, key) {
    // if (key === '__esModule' && !keyPath.length) return false;
    // if (key === Symbol.toPrimitive || loaded()) {
    // return () => _export(load(), keyPath);
    // }
    // return recursiveProxy(load, loaded, keyPath.concat(key));
    // },
    // set(target, key, value) {
    // _export(load(), keyPath)[key] = value;
    // },
    // apply(target, context, args) {
    // return _export(load(), keyPath).apply(context, args);
    // },
    // construct(target, args, newTarget) {
    // return Reflect.construct(
    // _export(load(), keyPath),
    // args,
    // newTarget,
    // );
    // },
    // });
    // _require = (id) => {
    // const node = id.replace('node:', '');
    // if (node === 'process') return processShim;
    // if (builtinModules.includes(node)) return require(node);
    // const cached = requireCache.get(id);
    // if (cached) return cached;
    // let loaded = false;
    // let mod;
    // const getMod = () =>
    // loaded ? mod : ((loaded = true), (mod = require(id)));
    // return recursiveProxy(getMod, () => loaded);
    // };
    // }

    const _eval = (code) => {
    return require('node:vm').runInNewContext(
    code,
    {
    ...global,
    require: _require,
    process: processShim,
    },
    {filename},
    );
    // const {Script} = require('node:vm');
    // const script = new Script(code, {
    // filename,
    // cachedData: hash && cacheStore?.get(filename, hash),
    // produceCachedData: Boolean(hash),
    // });
    // if (hash) cacheStore?.set(filename, hash, script.cachedData);
    // return script.runInNewContext({
    // ...global,
    // require: _require,
    // process: processShim,
    // });
    };
    const prom = compiledCode
    ? Promise.resolve(compiledCode).then(_eval)
    : isModule
    ? _import(filename)
    : readFile(filename).then(_eval);
    return prom.catch(console.error).then(end);
    }
    // } else if (process.env.POOL) {
    // if (process.env.POOL) {
    // worker = workerPool.get();
    // worker.postMessage([isModule, filename, args, $env(env), compiledCode]);
    // } else {
    @@ -495,25 +362,22 @@ async function runTs(script, args, env) {
    );
    end();
    // process.setSourceMapsEnabled(true);
    return runNode(script, args, env, code, result.outputFiles[0].hash);
    return runNode(script, args, env, code);
    }

    const BIN = {
    'ts-node': function (args, env) {
    'ts-node': (args, env) => {
    let script;
    const remainingArgs = args.filter((arg) => {
    if (arg[0] !== '-' && !script) {
    script = arg;
    return false;
    }
    return true;
    if (arg[0] === '-' || script) return true;
    script = arg;
    return false;
    });
    return runTs(script, remainingArgs, env);
    },
    };

    async function runBin(bin, args, env) {
    // console.log('RUN BIN', {bin, args, env});
    if (Object.prototype.hasOwnProperty.call(BIN, bin)) {
    return BIN[bin](args, env);
    }
    @@ -553,16 +417,10 @@ async function runScript(scriptName, args, env) {
    const parsed = parseShell(scriptWithArgs);
    if (DEBUG.parsing) console.log('parsed: ', scriptWithArgs, parsed);
    done();
    // console.log('RUN SCRIPT', {scriptName, scriptWithArgs, args, env});
    // console.log(require('node:util').inspect(parsed, {depth: 4, colors: true}));
    return run(parsed, env);
    }

    // let cacheStore;

    (async () => {
    // if (process.env.COMPILECACHE) installCache();

    const script = process.argv[2];
    if (!script) panic('No script name provided.');
    const start = performance.now();
    @@ -582,205 +440,3 @@ async function runScript(scriptName, args, env) {
    }
    }
    })();

    /*
    function installCache() {
    console.log('install cache');
    const Module = require('node:module');
    const crypto = require('node:crypto');
    const fs = require('node:fs');
    const path = require('node:path');
    const vm = require('node:vm');
    class FileSystemBlobStore {
    constructor(directory) {
    this._blobFilename = path.join(directory, 'BLOB');
    this._mapFilename = path.join(directory, 'MAP');
    try {
    this._storedBlob = fs.readFileSync(this._blobFilename);
    this._storedMap = new Map(
    Object.entries(JSON.parse(fs.readFileSync(this._mapFilename))),
    );
    } catch (e) {
    this._storedBlob = Buffer.alloc(0);
    this._storedMap = new Map();
    }
    this._dirty = false;
    this._memoryBlobs = new Map();
    this._invalidationKeys = new Map();
    }
    has(key, invalidationKey) {
    if (this._memoryBlobs.has(key)) {
    return this._invalidationKeys.get(key) === invalidationKey;
    } else if (this._storedMap.has(key)) {
    return this._storedMap.get(key)[0] === invalidationKey;
    }
    return false;
    }
    get(key, invalidationKey) {
    if (this._memoryBlobs.has(key)) {
    if (this._invalidationKeys.get(key) === invalidationKey) {
    return this._memoryBlobs.get(key);
    }
    } else if (this._storedMap.has(key)) {
    const mapping = this._storedMap.get(key);
    if (mapping[0] === invalidationKey) {
    return this._storedBlob.slice(mapping[1], mapping[2]);
    }
    }
    }
    set(key, invalidationKey, buffer) {
    this._invalidationKeys.set(key, invalidationKey);
    this._memoryBlobs.set(key, buffer);
    this._dirty = true;
    }
    delete(key) {
    const d1 = this._memoryBlobs.delete(key);
    const d2 = this._invalidationKeys.delete(key);
    const d3 = this._storedMap.delete(key);
    if (d1 || d2 || d3) this._dirty = true;
    }
    save() {
    if (!this._dirty) return;
    const dir = path.dirname(this._blobFilename);
    const lock = path.join(dir, 'LOCK');
    try {
    fs.mkdirSync(dir, {recursive: true});
    } catch (_) {}
    try {
    fs.writeFileSync(lock, 'LOCK', {flag: 'wx'});
    } catch (_) {
    return;
    }
    const dump = this._getDump();
    try {
    fs.writeFileSync(this._blobFilename, Buffer.concat(dump[0]));
    fs.writeFileSync(this._mapFilename, JSON.stringify(dump[1]));
    } finally {
    fs.unlinkSync(lock);
    }
    }
    _getDump() {
    const buffers = [];
    const newMap = {};
    let offset = 0;
    function push(key, invalidationKey, buffer) {
    buffers.push(buffer);
    newMap[key] = [invalidationKey, offset, offset + buffer.length];
    offset += buffer.length;
    }
    for (const [key, buffer] of this._memoryBlobs.entries()) {
    const invalidationKey = this._invalidationKeys.get(key);
    push(key, invalidationKey, buffer);
    }
    for (const [key, mapping] of this._storedMap.entries()) {
    const buffer = this._storedBlob.slice(mapping[1], mapping[2]);
    push(key, mapping[0], buffer);
    }
    return [buffers, newMap];
    }
    }
    cacheStore = new FileSystemBlobStore(
    path.join(
    process.cwd(),
    'node_modules/.runv8cache',
    process.arch + process.versions.v8,
    ),
    );
    // function compileModule(filename, content) {
    // const contLen = content.length;
    // if (contLen >= 2) {
    // if (content.charCodeAt(0) === 35 && content.charCodeAt(1) === 33) {
    // if (contLen === 2) {
    // // Exact match
    // content = '';
    // } else {
    // // Find end of shebang line and slice it off
    // let i = 2;
    // for (; i < contLen; ++i) {
    // const code = content.charCodeAt(i);
    // if (code === 10 || code === 13) break;
    // }
    // if (i === contLen) {
    // content = '';
    // } else {
    // // Note that this actually includes the newline character(s) in the
    // // new output. This duplicates the behavior of the regular
    // // expression that was previously used to replace the shebang line
    // content = content.slice(i);
    // }
    // }
    // }
    // }
    // const invalidationKey = crypto
    // .createHash('sha1')
    // .update(content, 'utf8')
    // .digest('hex');
    // const script = new vm.Script(Module.wrap(content), {
    // filename,
    // lineOffset: 0,
    // displayErrors: true,
    // cachedData: cacheStore.get(filename, invalidationKey),
    // produceCachedData: true,
    // });
    // if (script.cachedDataProduced) {
    // cacheStore.set(filename, invalidationKey, script.cachedData);
    // } else if (script.cachedDataRejected) {
    // cacheStore.delete(filename);
    // }
    // const compiledWrapper = script.runInThisContext({
    // filename,
    // lineOffset: 0,
    // columnOffset: 0,
    // displayErrors: true,
    // });
    // return compiledWrapper;
    // }
    // Module.prototype._compile = function (content, filename) {
    // const mod = this;
    // function require(id) {
    // return mod.require(id);
    // }
    // require.resolve = function (request, options) {
    // return Module._resolveFilename(request, mod, false, options);
    // };
    // require.resolve.paths = (req) => Module._resolveLookupPaths(req, mod, true);
    // require.main = process.mainModule;
    // require.extensions = Module._extensions;
    // require.cache = Module._cache;
    // return compileModule(filename, content).call(
    // mod.exports,
    // mod.exports,
    // require,
    // mod,
    // filename,
    // path.dirname(filename),
    // process,
    // global,
    // );
    // };
    process.once('exit', () => cacheStore.save());
    }
    */
    202 changes: 0 additions & 202 deletions v8-compile-cache.js
    View file
    Original file line number Diff line number Diff line change
    @@ -1,202 +0,0 @@
    {
    const Module = require('node:module');
    const crypto = require('node:crypto');
    const fs = require('node:fs');
    const path = require('node:path');
    const vm = require('node:vm');

    class FileSystemBlobStore {
    constructor(directory) {
    this._blobFilename = path.join(directory, 'BLOB');
    this._mapFilename = path.join(directory, 'MAP');
    try {
    this._storedBlob = fs.readFileSync(this._blobFilename);
    this._storedMap = new Map(
    Object.entries(JSON.parse(fs.readFileSync(this._mapFilename))),
    );
    } catch (e) {
    this._storedBlob = Buffer.alloc(0);
    this._storedMap = new Map();
    }
    this._dirty = false;
    this._memoryBlobs = new Map();
    this._invalidationKeys = new Map();
    }

    has(key, invalidationKey) {
    if (this._memoryBlobs.has(key)) {
    return this._invalidationKeys.get(key) === invalidationKey;
    } else if (this._storedMap.has(key)) {
    return this._storedMap.get(key)[0] === invalidationKey;
    }
    return false;
    }

    get(key, invalidationKey) {
    if (this._memoryBlobs.has(key)) {
    if (this._invalidationKeys.get(key) === invalidationKey) {
    return this._memoryBlobs.get(key);
    }
    } else if (this._storedMap.has(key)) {
    const mapping = this._storedMap.get(key);
    if (mapping[0] === invalidationKey) {
    return this._storedBlob.slice(mapping[1], mapping[2]);
    }
    }
    }

    set(key, invalidationKey, buffer) {
    this._invalidationKeys.set(key, invalidationKey);
    this._memoryBlobs.set(key, buffer);
    this._dirty = true;
    }

    delete(key) {
    const d1 = this._memoryBlobs.delete(key);
    const d2 = this._invalidationKeys.delete(key);
    const d3 = this._storedMap.delete(key);
    if (d1 || d2 || d3) this._dirty = true;
    }

    save() {
    if (!this._dirty) return;
    const dump = this._getDump();
    const blobToStore = Buffer.concat(dump[0]);
    const mapToStore = JSON.stringify(dump[1]);
    const lock = path.join(path.dirname(this._blobFilename), 'LOCK');

    try {
    fs.mkdirSync(lock, {recursive: true});
    fs.writeFileSync(this._lockFilename, 'LOCK', {flag: 'wx'});
    } catch (error) {
    // Swallow the exception if we fail to acquire the lock.
    return false;
    }

    try {
    fs.writeFileSync(this._blobFilename, blobToStore);
    fs.writeFileSync(this._mapFilename, mapToStore);
    } finally {
    fs.unlinkSync(lock);
    }

    return true;
    }

    _getDump() {
    const buffers = [];
    const newMap = {};
    let offset = 0;

    function push(key, invalidationKey, buffer) {
    buffers.push(buffer);
    newMap[key] = [invalidationKey, offset, offset + buffer.length];
    offset += buffer.length;
    }

    for (const [key, buffer] of this._memoryBlobs.entries()) {
    const invalidationKey = this._invalidationKeys[key];
    push(key, invalidationKey, buffer);
    }

    for (const [key, mapping] of this._storedMap.entries()) {
    const buffer = this._storedBlob.slice(mapping[1], mapping[2]);
    push(key, mapping[0], buffer);
    }

    return [buffers, newMap];
    }
    }

    const cacheStore = new FileSystemBlobStore(
    path.join(
    process.cwd(),
    'node_modules/.runv8cache',
    process.arch + process.versions.v8,
    ),
    );

    function compileModule(filename, content) {
    const contLen = content.length;
    if (contLen >= 2) {
    if (
    content.charCodeAt(0) === 35 /* #*/ &&
    content.charCodeAt(1) === 33 /* !*/
    ) {
    if (contLen === 2) {
    // Exact match
    content = '';
    } else {
    // Find end of shebang line and slice it off
    let i = 2;
    for (; i < contLen; ++i) {
    const code = content.charCodeAt(i);
    if (code === 10 /* \n*/ || code === 13 /* \r*/) break;
    }
    if (i === contLen) {
    content = '';
    } else {
    // Note that this actually includes the newline character(s) in the
    // new output. This duplicates the behavior of the regular
    // expression that was previously used to replace the shebang line
    content = content.slice(i);
    }
    }
    }
    }

    const invalidationKey = crypto
    .createHash('sha1')
    .update(content, 'utf8')
    .digest('hex');

    const script = new vm.Script(Module.wrap(content), {
    filename,
    lineOffset: 0,
    displayErrors: true,
    cachedData: cacheStore.get(filename, invalidationKey),
    produceCachedData: true,
    });

    if (script.cachedDataProduced) {
    cacheStore.set(filename, invalidationKey, script.cachedData);
    } else if (script.cachedDataRejected) {
    cacheStore.delete(filename);
    }

    const compiledWrapper = script.runInThisContext({
    filename,
    lineOffset: 0,
    columnOffset: 0,
    displayErrors: true,
    });

    return compiledWrapper;
    }

    Module.prototype._compile = function (content, filename) {
    const mod = this;
    function require(id) {
    return mod.require(id);
    }
    require.resolve = function (request, options) {
    return Module._resolveFilename(request, mod, false, options);
    };
    require.resolve.paths = (req) => Module._resolveLookupPaths(req, mod, true);
    require.main = process.mainModule;
    require.extensions = Module._extensions;
    require.cache = Module._cache;
    return compileModule(filename, content).call(
    mod.exports,
    mod.exports,
    require,
    mod,
    filename,
    path.dirname(filename),
    process,
    global,
    );
    };

    process.once('exit', () => cacheStore.save());
    }
  7. developit created this gist May 31, 2024.
    786 changes: 786 additions & 0 deletions run
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,786 @@
    #!/usr/bin/env node

    /* eslint-disable no-param-reassign, no-fallthrough, guard-for-in, no-nested-ternary */

    const {Worker} = require('node:worker_threads');
    const path = require('node:path');

    const DEBUG = {parsing: false, timing: false};
    const debugEnv = process.env.DEBUG;
    if (debugEnv) {
    for (const name in DEBUG) {
    DEBUG[name] =
    debugEnv === 'true' || debugEnv === '1' || debugEnv.includes(name);
    }
    }
    const readFile = (file) => require('node:fs/promises').readFile(file, 'utf-8');

    // process.env.NODE_OPTIONS = `--no-addons --no-deprecation --no-force-async-hooks-checks --no-global-search-paths ${
    // process.env.NODE_OPTIONS || ''
    // }`;

    const $env = (env) => (env ? {...process.env, ...env} : process.env);

    let markId = 0;
    function measure(name, parent = '', depth = 0) {
    const start = `${parent}:${name}:${++markId}`;
    performance.mark(start);
    function end() {
    performance.measure(name, {
    start,
    detail: {depth, parent},
    });
    }
    end.submeasure = (sub) => measure(sub, start, depth + 1);
    return end;
    }

    const color = (code, end) => (str) => `\x1B[${code}m${str}\x1B[${end}m`;
    const bold = color(1, 22);
    const dim = color(2, 22);
    const red = color(31, 39);
    const panic = (message) => {
    console.log(red(bold('Error: ') + message));
    process.exit(1);
    };

    function json5(json) {
    try {
    return JSON.parse(json);
    } catch (_2) {
    return (0, eval)(`(${json})`);
    }
    }

    const endPkgParse = measure('parse package.json');
    let pkg;
    const pkgPromise = readFile('package.json').then((json) => {
    pkg = json5(json);
    endPkgParse();
    return pkg;
    });

    const INITIAL = 0;
    const COMMAND = 1;
    const ARGS = 2;

    function parseShell(script) {
    const tokenizer = /(if|then|fi|&&|\|\||[ ;"'`()=])/g;
    script += ' ';
    let token;
    let index = 0;
    const stack = [];
    const logic = [];
    let lastOp;
    let context = INITIAL;
    let lastEnv;
    let buf = '';
    let command = {name: '', args: [], env: {}};
    const root = [];
    let seq = root;
    const expectedOps = {
    ')': '(',
    // then: 'if',
    fi: 'if',
    };
    const commit = () => {
    // console.log({context, buf, lastEnv});
    if (context === INITIAL) {
    if (lastEnv) {
    if (!command) seq.push((command = {name: '', args: [], env: {}}));
    command.env[lastEnv] = buf;
    lastEnv = undefined;
    } else if (buf) {
    if (!command) seq.push((command = {name: '', args: [], env: {}}));
    command.name = buf;
    context = ARGS;
    }
    } else if (context === ARGS) {
    command.args.push(buf);
    }
    buf = '';
    };
    while ((token = tokenizer.exec(script))) {
    let op = token[0];
    // console.log(token);
    buf += script.substring(index, token.index);
    index = tokenizer.lastIndex;
    switch (op) {
    case '&&':
    case '||':
    if (lastOp) continue;
    seq.push({type: op});
    case ';':
    // if (lastOp === 'if') continue;
    if (lastOp) continue;
    commit();
    command = null;
    context = INITIAL;
    break;
    case ' ':
    // console.log('space', {context, buf, lastOp, lastEnv});
    if (!buf) continue;
    if (lastOp) {
    if (buf) buf += ' ';
    continue;
    }
    commit();
    break;
    case '=':
    if (context === INITIAL && !lastOp) {
    lastEnv = buf;
    buf = '';
    } else {
    buf += op;
    }
    break;
    case 'if':
    logic.push({
    type: 'conditional',
    test: [],
    consequent: [],
    alternate: [],
    });
    seq = logic.at(-1)?.test;
    // seq.push((command = {name: '', args: [], env: {}}));
    command = null;
    context = INITIAL;
    break;
    case 'then':
    seq = logic.at(-1)?.consequent;
    // seq.push((command = {name: '', args: [], env: {}}));
    command = null;
    context = INITIAL;
    break;
    case 'else':
    seq = logic.at(-1)?.alternate;
    // seq.push((command = {name: '', args: [], env: {}}));
    command = null;
    context = INITIAL;
    break;
    case 'fi':
    seq = root;
    if (logic.length) seq.push(logic.pop());
    break;
    case '(':
    case ')':
    case '"':
    case `'`:
    case '`':
    buf += op;
    default:
    // if (op === 'then') {
    // seq.push({type: 'if'});
    // seq.push((command = {name: '', args: [], env: {}}));
    // continue;
    // }
    if (op in expectedOps) {
    const start = expectedOps[op];
    if (lastOp !== start) {
    // console.log({context, stack, buf, op, start, lastOp, seq});
    const line = `${script}\n${'-'.repeat(token.index)}^`;
    panic(`unexpected "${op}"\n${dim(line)}`);
    }
    op = start;
    }
    if (lastOp === op) {
    stack.pop();
    lastOp = stack.at(-1);
    } else {
    stack.push(op);
    lastOp = op;
    // continue;
    }
    break;
    }
    }
    if (command && seq.at(-1) !== command) seq.push(command);
    return seq;
    }

    async function run(seq, env = {}) {
    let prev = true;
    for (const node of seq) {
    switch (node.type) {
    case '&&':
    if (!prev) return;
    break;
    case '||':
    if (prev) return prev;
    break;
    case 'conditional':
    if (await run(node.test, env)) {
    prev = await run(node.consequent, env);
    // await run(node.consequent, env);
    } else if (node.alternate) {
    prev = await run(node.alternate, env);
    // await run(node.alternate, env);
    }
    break;
    default:
    if (node.name === '$npm_execpath' || node.name === 'npm') {
    let script = node.args.shift();
    if (script === 'run') script = node.args.shift();
    prev = await runScript(script, node.args, {...env, ...node.env});
    } else if (node.name === 'node') {
    const script = node.args.shift();
    prev = await runNode(script, node.args, {...env, ...node.env});
    } else if (node.name === 'test') {
    const a1 = node.args[0].replace(
    /$([a-z0-9_]+)/g,
    (_, c) => process.env[c] ?? _,
    );
    const a2 = node.args[2].replace(
    /$([a-z0-9_]+)/g,
    (_, c) => process.env[c] ?? _,
    );
    if (node.args[1] === '=') prev = a1 === a2;
    else if (node.args[1] === '!=') prev = a1 !== a2;
    } else if (node.name) {
    // panic(`Unknown command ${node.name} (${node.args})`);
    prev = await runBin(node.name, node.args, node.env);
    } else {
    panic(`Unknown node ${JSON.stringify(node)}`);
    }
    }
    }
    return prev;
    }

    const workerShim = `
    require('worker_threads').parentPort.once('message', ([mod, filename, args, env, compiledCode]) => {
    process.argv.splice(2, process.argv.length, ...args);
    Object.assign(process.env, env);
    require('worker_threads').parentPort.postMessage('done');
    if (mod && !compiledCode) return import(filename);
    const code = compiledCode || require('fs').readFileSync(filename, 'utf-8');
    require('vm').runInThisContext(code, {filename});
    });
    `;
    const workerPool = {
    workers: [],
    get: () => workerPool.workers.pop() ?? workerPool.create(),
    create: () => new Worker(workerShim, {eval: true}),
    };
    if (!process.env.INLINE) workerPool.workers.push(workerPool.create());

    const pkgCache = new Map();
    function getPkg(filename) {
    const index = filename.lastIndexOf('/node_modules/');
    if (index === -1) return pkgPromise;
    const pkgName = filename.slice(index + 14).match(/^(@[^/]+\/)?[^/]+/g)[0];
    const pkgFile = `${filename.slice(0, index + 14)}${pkgName}/package.json`;
    const cached = pkgCache.get(pkgFile);
    if (cached) return cached;
    const prom = readFile(pkgFile).then(json5);
    pkgCache.set(pkgFile, prom);
    return prom;
    }

    async function runNode(script, args, env, compiledCode, hash) {
    // if (process.env.FORCE_ESBUILD && compiledCode === undefined) {
    // return runTs(script, args, env);
    // }
    const scriptId = `node(${script} ${args.join(' ')})`;
    const done = measure(scriptId);
    const filename = path.resolve(script);

    const isModule =
    filename.endsWith('.mjs') ||
    ((await getPkg(filename)).type === 'module' && !filename.endsWith('.cjs'));

    return new Promise((resolve, reject) => {
    // if (process.env.FORK) {
    // const {fork} = require('node:child_process');
    // return fork(script, args, {env, stdio: 'inherit'})
    // .once('error', reject)
    // .once('exit', resolve);
    // }

    const end = done.submeasure('worker start');
    // let worker;

    if (process.env.INLINE) {
    /** @type {typeof process} */
    const processShim = {
    ...process,
    argv: [process.argv[0], filename, ...args],
    exitCode: 0,
    exit(code) {
    processShim.exitCode = code;
    done();
    resolve(processShim.exitCode);
    },
    };

    const _import = async (id) => {
    const oldArgv = process.argv;
    const oldExit = process.exit;
    process.argv = processShim.argv;
    process.exit = processShim.exit;
    try {
    return await import(id);
    } finally {
    process.argv = oldArgv;
    process.exit = oldExit;
    }
    };

    const _require = (id) => {
    if (id === 'process' || id === 'node:process') return processShim;
    return require(id);
    };

    // if (process.env.LAZYREQUIRE) {
    // const {builtinModules} = require('node:module');
    // const requireCache = new Map();
    // const dlv = (obj, keyPath) => {
    // for (const key of keyPath) obj = obj?.[key];
    // return obj;
    // };
    // const _export = (root, keyPath) =>
    // dlv(root, keyPath) ?? dlv(root.default, keyPath);
    // const recursiveProxy = (load, loaded, keyPath = []) =>
    // new Proxy(function () {}, {
    // get(target, key) {
    // if (key === '__esModule' && !keyPath.length) return false;
    // if (key === Symbol.toPrimitive || loaded()) {
    // return () => _export(load(), keyPath);
    // }
    // return recursiveProxy(load, loaded, keyPath.concat(key));
    // },
    // set(target, key, value) {
    // _export(load(), keyPath)[key] = value;
    // },
    // apply(target, context, args) {
    // return _export(load(), keyPath).apply(context, args);
    // },
    // construct(target, args, newTarget) {
    // return Reflect.construct(
    // _export(load(), keyPath),
    // args,
    // newTarget,
    // );
    // },
    // });
    // _require = (id) => {
    // const node = id.replace('node:', '');
    // if (node === 'process') return processShim;
    // if (builtinModules.includes(node)) return require(node);
    // const cached = requireCache.get(id);
    // if (cached) return cached;
    // let loaded = false;
    // let mod;
    // const getMod = () =>
    // loaded ? mod : ((loaded = true), (mod = require(id)));
    // return recursiveProxy(getMod, () => loaded);
    // };
    // }

    const _eval = (code) => {
    return require('node:vm').runInNewContext(
    code,
    {
    ...global,
    require: _require,
    process: processShim,
    },
    {filename},
    );
    // const {Script} = require('node:vm');
    // const script = new Script(code, {
    // filename,
    // cachedData: hash && cacheStore?.get(filename, hash),
    // produceCachedData: Boolean(hash),
    // });
    // if (hash) cacheStore?.set(filename, hash, script.cachedData);
    // return script.runInNewContext({
    // ...global,
    // require: _require,
    // process: processShim,
    // });
    };
    const prom = compiledCode
    ? Promise.resolve(compiledCode).then(_eval)
    : isModule
    ? _import(filename)
    : readFile(filename).then(_eval);
    return prom.catch(console.error).then(end);
    }
    // } else if (process.env.POOL) {
    // worker = workerPool.get();
    // worker.postMessage([isModule, filename, args, $env(env), compiledCode]);
    // } else {
    // const scriptStr = JSON.stringify(filename);
    // const shim = `{
    // process.argv[1] = ${scriptStr};
    // ${
    // compiledCode
    // ? `require('vm').runInThisContext(${JSON.stringify(
    // compiledCode,
    // )}, {filename: ${scriptStr}});`
    // : isModule
    // ? `import(${scriptStr});`
    // : `require('vm').runInThisContext(require('fs').readFileSync(${scriptStr}, 'utf-8'), {filename: ${scriptStr}});`
    // }
    // require('worker_threads').parentPort.postMessage(0);
    // }`;
    // worker = new Worker(shim, {
    // env: $env(env),
    // argv: args,
    // eval: true,
    // // resourceLimits: {
    // // maxOldGenerationSizeMb: 9000,
    // // maxYoungGenerationSizeMb: 9000,
    // // },
    // });
    // }
    const worker = workerPool.get();
    worker.postMessage([isModule, filename, args, $env(env), compiledCode]);
    end();
    worker.once('message', done.submeasure('worker init'));
    worker.on('error', (err) => {
    let stack = err.stack;
    try {
    const ref = err.stack?.match(/^\s*at .*?\((\/[^)]*?):(\d+):(\d+)\)?$/m);
    if (ref) {
    const [, file, lineNo, colNo] = ref;
    const code = require('node:fs').readFileSync(file, 'utf-8');
    const loc = `${file}:${lineNo}:${colNo}`;
    const line = code.split('\n')[lineNo - 1];
    stack = `\n on ${loc}:\n ${line}\n ${'-'.repeat(colNo | 0)}^`;
    }
    } catch (_) {
    // ignore, use native stack
    }
    console.error(red(`${bold('Error')}: ${err}${dim(stack)}`));
    done();
    reject(err);
    });
    worker.once('exit', (exitCode) => {
    done();
    resolve(exitCode);
    });
    });
    }

    async function runTs(script, args, env) {
    const end = measure(`esbuild(${script})`);
    const {build} = require('esbuild');
    const result = await build({
    bundle: true,
    entryPoints: [script],
    target: 'node20',
    platform: 'node',
    format: 'cjs',
    packages: 'external',
    write: false,
    treeShaking: true,
    nodePaths: [],
    });
    if (result.errors.length) {
    panic(`TS build of ${script} failed:\n${result.errors.join('\n')}`);
    }
    let code = result.outputFiles[0].text;

    // move top-level requires to callsites so they only get evaluated when surrounding code is executed
    const idents = {};
    code = code.replace(
    /(?:^var (import_[a-zA-Z0-9_$]+) = ((?:__toESM\()?require\("[^"]+"\)\)?);?$|\b(import_[a-zA-Z0-9_$]+)\b)/gm,
    (_, ident, req, ident2) => {
    if (ident2) return idents[ident2] ?? ident2;
    idents[ident] = req;
    return '';
    },
    );
    end();
    // process.setSourceMapsEnabled(true);
    return runNode(script, args, env, code, result.outputFiles[0].hash);
    }

    const BIN = {
    'ts-node': function (args, env) {
    let script;
    const remainingArgs = args.filter((arg) => {
    if (arg[0] !== '-' && !script) {
    script = arg;
    return false;
    }
    return true;
    });
    return runTs(script, remainingArgs, env);
    },
    };

    async function runBin(bin, args, env) {
    // console.log('RUN BIN', {bin, args, env});
    if (Object.prototype.hasOwnProperty.call(BIN, bin)) {
    return BIN[bin](args, env);
    }
    try {
    const done = measure(`resolve-bin(${bin} ${args.join(' ')})`);
    const code = await readFile(`node_modules/.bin/${bin}`);
    // exec node "$basedir/../vite/bin/vite.js" "$@"
    const match = code.match(/exec node +"([^"]+)" "\$@"/)[1];
    if (!match) panic(`Unknown format at node_modules/.bin/${bin}`);
    const binFile = path.resolve(
    match.replace('$basedir', './node_modules/.bin'),
    );
    const binRel = path.relative('.', binFile);
    done();
    return runNode(binRel, args, env);
    } catch (err) {
    const done = measure(`run-bin(${bin} ${args.join(' ')})`);
    console.error(`failed to run "${bin}" seamlessly:`, err);
    return new Promise((resolve) => {
    const {spawn} = require('node:child_process');
    const proc = spawn(bin, args, {env: $env(env), stdio: 'inherit'});
    // proc.on('error', reject);
    proc.once('exit', (exitCode) => {
    done();
    resolve(exitCode);
    });
    });
    }
    }

    async function runScript(scriptName, args, env) {
    const script = (await pkgPromise).scripts[scriptName];
    if (!script) panic(`Unknown script "${scriptName}".`);
    let scriptWithArgs = script;
    for (const arg of args) scriptWithArgs += ` ${arg}`;
    const done = measure('parse');
    const parsed = parseShell(scriptWithArgs);
    if (DEBUG.parsing) console.log('parsed: ', scriptWithArgs, parsed);
    done();
    // console.log('RUN SCRIPT', {scriptName, scriptWithArgs, args, env});
    // console.log(require('node:util').inspect(parsed, {depth: 4, colors: true}));
    return run(parsed, env);
    }

    // let cacheStore;

    (async () => {
    // if (process.env.COMPILECACHE) installCache();

    const script = process.argv[2];
    if (!script) panic('No script name provided.');
    const start = performance.now();
    try {
    // eslint-disable-next-line require-atomic-updates
    process.exitCode = await runScript(script, process.argv.slice(3));
    } finally {
    let worker;
    while ((worker = workerPool.workers?.pop())) worker.terminate();
    if (DEBUG.timing) {
    console.log(bold(dim(`Done in ${(performance.now() - start) | 0}ms:`)));
    const dur = (ms) => (ms < 1 ? `${(ms * 1000) | 0}µs` : `${ms | 0}ms`);
    for (const entry of performance.getEntriesByType('measure')) {
    const pfx = entry.detail.depth ? ' '.repeat(entry.detail.depth) : '';
    console.log(dim(`${pfx}⏱ ${dur(entry.duration)}: ${entry.name}`));
    }
    }
    }
    })();

    /*
    function installCache() {
    console.log('install cache');
    const Module = require('node:module');
    const crypto = require('node:crypto');
    const fs = require('node:fs');
    const path = require('node:path');
    const vm = require('node:vm');
    class FileSystemBlobStore {
    constructor(directory) {
    this._blobFilename = path.join(directory, 'BLOB');
    this._mapFilename = path.join(directory, 'MAP');
    try {
    this._storedBlob = fs.readFileSync(this._blobFilename);
    this._storedMap = new Map(
    Object.entries(JSON.parse(fs.readFileSync(this._mapFilename))),
    );
    } catch (e) {
    this._storedBlob = Buffer.alloc(0);
    this._storedMap = new Map();
    }
    this._dirty = false;
    this._memoryBlobs = new Map();
    this._invalidationKeys = new Map();
    }
    has(key, invalidationKey) {
    if (this._memoryBlobs.has(key)) {
    return this._invalidationKeys.get(key) === invalidationKey;
    } else if (this._storedMap.has(key)) {
    return this._storedMap.get(key)[0] === invalidationKey;
    }
    return false;
    }
    get(key, invalidationKey) {
    if (this._memoryBlobs.has(key)) {
    if (this._invalidationKeys.get(key) === invalidationKey) {
    return this._memoryBlobs.get(key);
    }
    } else if (this._storedMap.has(key)) {
    const mapping = this._storedMap.get(key);
    if (mapping[0] === invalidationKey) {
    return this._storedBlob.slice(mapping[1], mapping[2]);
    }
    }
    }
    set(key, invalidationKey, buffer) {
    this._invalidationKeys.set(key, invalidationKey);
    this._memoryBlobs.set(key, buffer);
    this._dirty = true;
    }
    delete(key) {
    const d1 = this._memoryBlobs.delete(key);
    const d2 = this._invalidationKeys.delete(key);
    const d3 = this._storedMap.delete(key);
    if (d1 || d2 || d3) this._dirty = true;
    }
    save() {
    if (!this._dirty) return;
    const dir = path.dirname(this._blobFilename);
    const lock = path.join(dir, 'LOCK');
    try {
    fs.mkdirSync(dir, {recursive: true});
    } catch (_) {}
    try {
    fs.writeFileSync(lock, 'LOCK', {flag: 'wx'});
    } catch (_) {
    return;
    }
    const dump = this._getDump();
    try {
    fs.writeFileSync(this._blobFilename, Buffer.concat(dump[0]));
    fs.writeFileSync(this._mapFilename, JSON.stringify(dump[1]));
    } finally {
    fs.unlinkSync(lock);
    }
    }
    _getDump() {
    const buffers = [];
    const newMap = {};
    let offset = 0;
    function push(key, invalidationKey, buffer) {
    buffers.push(buffer);
    newMap[key] = [invalidationKey, offset, offset + buffer.length];
    offset += buffer.length;
    }
    for (const [key, buffer] of this._memoryBlobs.entries()) {
    const invalidationKey = this._invalidationKeys.get(key);
    push(key, invalidationKey, buffer);
    }
    for (const [key, mapping] of this._storedMap.entries()) {
    const buffer = this._storedBlob.slice(mapping[1], mapping[2]);
    push(key, mapping[0], buffer);
    }
    return [buffers, newMap];
    }
    }
    cacheStore = new FileSystemBlobStore(
    path.join(
    process.cwd(),
    'node_modules/.runv8cache',
    process.arch + process.versions.v8,
    ),
    );
    // function compileModule(filename, content) {
    // const contLen = content.length;
    // if (contLen >= 2) {
    // if (content.charCodeAt(0) === 35 && content.charCodeAt(1) === 33) {
    // if (contLen === 2) {
    // // Exact match
    // content = '';
    // } else {
    // // Find end of shebang line and slice it off
    // let i = 2;
    // for (; i < contLen; ++i) {
    // const code = content.charCodeAt(i);
    // if (code === 10 || code === 13) break;
    // }
    // if (i === contLen) {
    // content = '';
    // } else {
    // // Note that this actually includes the newline character(s) in the
    // // new output. This duplicates the behavior of the regular
    // // expression that was previously used to replace the shebang line
    // content = content.slice(i);
    // }
    // }
    // }
    // }
    // const invalidationKey = crypto
    // .createHash('sha1')
    // .update(content, 'utf8')
    // .digest('hex');
    // const script = new vm.Script(Module.wrap(content), {
    // filename,
    // lineOffset: 0,
    // displayErrors: true,
    // cachedData: cacheStore.get(filename, invalidationKey),
    // produceCachedData: true,
    // });
    // if (script.cachedDataProduced) {
    // cacheStore.set(filename, invalidationKey, script.cachedData);
    // } else if (script.cachedDataRejected) {
    // cacheStore.delete(filename);
    // }
    // const compiledWrapper = script.runInThisContext({
    // filename,
    // lineOffset: 0,
    // columnOffset: 0,
    // displayErrors: true,
    // });
    // return compiledWrapper;
    // }
    // Module.prototype._compile = function (content, filename) {
    // const mod = this;
    // function require(id) {
    // return mod.require(id);
    // }
    // require.resolve = function (request, options) {
    // return Module._resolveFilename(request, mod, false, options);
    // };
    // require.resolve.paths = (req) => Module._resolveLookupPaths(req, mod, true);
    // require.main = process.mainModule;
    // require.extensions = Module._extensions;
    // require.cache = Module._cache;
    // return compileModule(filename, content).call(
    // mod.exports,
    // mod.exports,
    // require,
    // mod,
    // filename,
    // path.dirname(filename),
    // process,
    // global,
    // );
    // };
    process.once('exit', () => cacheStore.save());
    }
    */
    202 changes: 202 additions & 0 deletions v8-compile-cache.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,202 @@
    {
    const Module = require('node:module');
    const crypto = require('node:crypto');
    const fs = require('node:fs');
    const path = require('node:path');
    const vm = require('node:vm');

    class FileSystemBlobStore {
    constructor(directory) {
    this._blobFilename = path.join(directory, 'BLOB');
    this._mapFilename = path.join(directory, 'MAP');
    try {
    this._storedBlob = fs.readFileSync(this._blobFilename);
    this._storedMap = new Map(
    Object.entries(JSON.parse(fs.readFileSync(this._mapFilename))),
    );
    } catch (e) {
    this._storedBlob = Buffer.alloc(0);
    this._storedMap = new Map();
    }
    this._dirty = false;
    this._memoryBlobs = new Map();
    this._invalidationKeys = new Map();
    }

    has(key, invalidationKey) {
    if (this._memoryBlobs.has(key)) {
    return this._invalidationKeys.get(key) === invalidationKey;
    } else if (this._storedMap.has(key)) {
    return this._storedMap.get(key)[0] === invalidationKey;
    }
    return false;
    }

    get(key, invalidationKey) {
    if (this._memoryBlobs.has(key)) {
    if (this._invalidationKeys.get(key) === invalidationKey) {
    return this._memoryBlobs.get(key);
    }
    } else if (this._storedMap.has(key)) {
    const mapping = this._storedMap.get(key);
    if (mapping[0] === invalidationKey) {
    return this._storedBlob.slice(mapping[1], mapping[2]);
    }
    }
    }

    set(key, invalidationKey, buffer) {
    this._invalidationKeys.set(key, invalidationKey);
    this._memoryBlobs.set(key, buffer);
    this._dirty = true;
    }

    delete(key) {
    const d1 = this._memoryBlobs.delete(key);
    const d2 = this._invalidationKeys.delete(key);
    const d3 = this._storedMap.delete(key);
    if (d1 || d2 || d3) this._dirty = true;
    }

    save() {
    if (!this._dirty) return;
    const dump = this._getDump();
    const blobToStore = Buffer.concat(dump[0]);
    const mapToStore = JSON.stringify(dump[1]);
    const lock = path.join(path.dirname(this._blobFilename), 'LOCK');

    try {
    fs.mkdirSync(lock, {recursive: true});
    fs.writeFileSync(this._lockFilename, 'LOCK', {flag: 'wx'});
    } catch (error) {
    // Swallow the exception if we fail to acquire the lock.
    return false;
    }

    try {
    fs.writeFileSync(this._blobFilename, blobToStore);
    fs.writeFileSync(this._mapFilename, mapToStore);
    } finally {
    fs.unlinkSync(lock);
    }

    return true;
    }

    _getDump() {
    const buffers = [];
    const newMap = {};
    let offset = 0;

    function push(key, invalidationKey, buffer) {
    buffers.push(buffer);
    newMap[key] = [invalidationKey, offset, offset + buffer.length];
    offset += buffer.length;
    }

    for (const [key, buffer] of this._memoryBlobs.entries()) {
    const invalidationKey = this._invalidationKeys[key];
    push(key, invalidationKey, buffer);
    }

    for (const [key, mapping] of this._storedMap.entries()) {
    const buffer = this._storedBlob.slice(mapping[1], mapping[2]);
    push(key, mapping[0], buffer);
    }

    return [buffers, newMap];
    }
    }

    const cacheStore = new FileSystemBlobStore(
    path.join(
    process.cwd(),
    'node_modules/.runv8cache',
    process.arch + process.versions.v8,
    ),
    );

    function compileModule(filename, content) {
    const contLen = content.length;
    if (contLen >= 2) {
    if (
    content.charCodeAt(0) === 35 /* #*/ &&
    content.charCodeAt(1) === 33 /* !*/
    ) {
    if (contLen === 2) {
    // Exact match
    content = '';
    } else {
    // Find end of shebang line and slice it off
    let i = 2;
    for (; i < contLen; ++i) {
    const code = content.charCodeAt(i);
    if (code === 10 /* \n*/ || code === 13 /* \r*/) break;
    }
    if (i === contLen) {
    content = '';
    } else {
    // Note that this actually includes the newline character(s) in the
    // new output. This duplicates the behavior of the regular
    // expression that was previously used to replace the shebang line
    content = content.slice(i);
    }
    }
    }
    }

    const invalidationKey = crypto
    .createHash('sha1')
    .update(content, 'utf8')
    .digest('hex');

    const script = new vm.Script(Module.wrap(content), {
    filename,
    lineOffset: 0,
    displayErrors: true,
    cachedData: cacheStore.get(filename, invalidationKey),
    produceCachedData: true,
    });

    if (script.cachedDataProduced) {
    cacheStore.set(filename, invalidationKey, script.cachedData);
    } else if (script.cachedDataRejected) {
    cacheStore.delete(filename);
    }

    const compiledWrapper = script.runInThisContext({
    filename,
    lineOffset: 0,
    columnOffset: 0,
    displayErrors: true,
    });

    return compiledWrapper;
    }

    Module.prototype._compile = function (content, filename) {
    const mod = this;
    function require(id) {
    return mod.require(id);
    }
    require.resolve = function (request, options) {
    return Module._resolveFilename(request, mod, false, options);
    };
    require.resolve.paths = (req) => Module._resolveLookupPaths(req, mod, true);
    require.main = process.mainModule;
    require.extensions = Module._extensions;
    require.cache = Module._cache;
    return compileModule(filename, content).call(
    mod.exports,
    mod.exports,
    require,
    mod,
    filename,
    path.dirname(filename),
    process,
    global,
    );
    };

    process.once('exit', () => cacheStore.save());
    }