internal/linker/link_node_modules.ts

/**
 * @fileoverview Creates a node_modules directory in the current working directory
 * and symlinks in the node modules needed to run a program.
 * This replaces the need for custom module resolution logic inside the process.
 */
import * as fs from 'fs';
import * as path from 'path';

// Run Bazel with --define=VERBOSE_LOGS=1 to enable this logging
const VERBOSE_LOGS = !!process.env['VERBOSE_LOGS'];

function log_verbose(...m: string[]) {
  if (VERBOSE_LOGS) console.error('[link_node_modules.js]', ...m);
}

function panic(m: string) {
  throw new Error(`Internal error! Please run again with
   --define=VERBOSE_LOG=1
and file an issue: https://github.com/bazelbuild/rules_nodejs/issues/new?template=bug_report.md
Include as much of the build output as you can without disclosing anything confidential.

  Error:
  ${m}
  `);
}

async function symlink(target: string, path: string) {
  log_verbose(`symlink( ${path} -> ${target} )`);
  // Use junction on Windows since symlinks require elevated permissions.
  // We only link to directories so junctions work for us.
  try {
    await fs.promises.symlink(target, path, 'junction');
  } catch (e) {
    if (e.code !== 'EEXIST') {
      throw e;
    }
    // We assume here that the path is already linked to the correct target.
    // Could add some logic that asserts it here, but we want to avoid an extra
    // filesystem access so we should only do it under some kind of strict mode.
  }

  if (VERBOSE_LOGS) {
    // Be verbose about creating a bad symlink
    // Maybe this should fail in production as well, but again we want to avoid
    // any unneeded file I/O
    if (!fs.existsSync(path)) {
      log_verbose(
          'ERROR\n***\nLooks like we created a bad symlink:' +
          `\n  pwd ${process.cwd()}\n  target ${target}\n***`);
    }
  }
}

/**
 * Resolve a root directory string to the actual location on disk
 * where node_modules was installed
 * @param root a string like 'npm/node_modules'
 */
function resolveRoot(root: string|undefined, runfiles: Runfiles) {
  // create a node_modules directory if no root
  // this will be the case if only first-party modules are installed
  if (!root) {
    if (!fs.existsSync('node_modules')) {
      log_verbose('no third-party packages; mkdir node_modules in ', process.cwd());
      fs.mkdirSync('node_modules');
    }
    return 'node_modules';
  }

  // If we got a runfilesManifest map, look through it for a resolution
  // This will happen if we are running a binary that had some npm packages
  // "statically linked" into its runfiles
  const fromManifest = runfiles.lookupDirectory(root);
  if (fromManifest) return fromManifest;

  // Account for Bazel --legacy_external_runfiles
  // which look like 'my_wksp/external/npm/node_modules'
  if (fs.existsSync(path.join('external', root))) {
    log_verbose('Found legacy_external_runfiles, switching root to', path.join('external', root));
    return path.join('external', root);
  }

  // The repository should be layed out in the parent directory
  // since bazel sets our working directory to the repository where the build is happening
  return path.join('..', root);
}

export class Runfiles {
  manifest: Map<string, string>|undefined;
  dir: string|undefined;

  constructor() {
    // If Bazel sets a variable pointing to a runfiles manifest,
    // we'll always use it.
    // Note that this has a slight performance implication on Mac/Linux
    // where we could use the runfiles tree already laid out on disk
    // but this just costs one file read for the external npm/node_modules
    // and one for each first-party module, not one per file.
    if (!!process.env['RUNFILES_MANIFEST_FILE']) {
      this.manifest = this.loadRunfilesManifest(process.env['RUNFILES_MANIFEST_FILE']!);
    } else if (!!process.env['RUNFILES_DIR']) {
      this.dir = path.resolve(process.env['RUNFILES_DIR']!);
    } else {
      panic(
          'Every node program run under Bazel must have a $RUNFILES_DIR or $RUNFILES_MANIFEST_FILE environment variable');
    }
    // Under --noenable_runfiles (in particular on Windows)
    // Bazel sets RUNFILES_MANIFEST_ONLY=1.
    // When this happens, we need to read the manifest file to locate
    // inputs
    if (process.env['RUNFILES_MANIFEST_ONLY'] === '1' && !process.env['RUNFILES_MANIFEST_FILE']) {
      log_verbose(`Workaround https://github.com/bazelbuild/bazel/issues/7994
                 RUNFILES_MANIFEST_FILE should have been set but wasn't.
                 falling back to using runfiles symlinks.
                 If you want to test runfiles manifest behavior, add
                 --spawn_strategy=standalone to the command line.`);
    }
  }

  lookupDirectory(dir: string): string|undefined {
    if (!this.manifest) return undefined;

    for (const [k, v] of this.manifest) {
      // Entry looks like
      // k: npm/node_modules/semver/LICENSE
      // v: /path/to/external/npm/node_modules/semver/LICENSE
      // calculate l = length(`/semver/LICENSE`)
      if (k.startsWith(dir)) {
        const l = k.length - dir.length;
        return v.substring(0, v.length - l);
      }
    }
  }


  /**
   * The runfiles manifest maps from short_path
   * https://docs.bazel.build/versions/master/skylark/lib/File.html#short_path
   * to the actual location on disk where the file can be read.
   *
   * In a sandboxed execution, it does not exist. In that case, runfiles must be
   * resolved from a symlink tree under the runfiles dir.
   * See https://github.com/bazelbuild/bazel/issues/3726
   */
  loadRunfilesManifest(manifestPath: string) {
    log_verbose(`using runfiles manifest ${manifestPath}`);

    const runfilesEntries = new Map();
    const input = fs.readFileSync(manifestPath, {encoding: 'utf-8'});

    for (const line of input.split('\n')) {
      if (!line) continue;
      const [runfilesPath, realPath] = line.split(' ');
      runfilesEntries.set(runfilesPath, realPath);
    }

    return runfilesEntries;
  }
}

// TypeScript lib.es5.d.ts has a mistake: JSON.parse does accept Buffer.
declare global {
  interface JSON {
    parse(b: {toString: () => string}): any;
  }
}

// There is no fs.promises.exists function because
// node core is of the opinion that exists is always too racey to rely on.
async function exists(p: string) {
  try {
    await fs.promises.stat(p)
    return true;
  } catch (e) {
    if (e.code === 'ENOENT') {
      return false;
    }
    throw e;
  }
}

export async function main(args: string[], runfiles: Runfiles) {
  if (!args || args.length < 1)
    throw new Error('link_node_modules.js requires one argument: modulesManifest path');

  const [modulesManifest] = args;
  let {root, modules, workspace} = JSON.parse(fs.readFileSync(modulesManifest));
  modules = modules || {};
  log_verbose(
      `module manifest: workspace ${workspace}, root ${root} with first-party packages\n`, modules);

  const rootDir = resolveRoot(root, runfiles);
  log_verbose('resolved root', root, 'to', rootDir);

  // Bazel starts actions with pwd=execroot/my_wksp
  const workspaceDir = path.resolve('.');

  // Convert from runfiles path
  // this_wksp/path/to/file OR other_wksp/path/to/file
  // to execroot path
  // path/to/file OR external/other_wksp/path/to/file
  function toWorkspaceDir(p: string) {
    if (p.startsWith(workspace + path.sep)) {
      return p.substring(workspace.length + 1);
    }
    return path.join('external', p);
  }

  // Create the $pwd/node_modules directory that node will resolve from
  await symlink(rootDir, 'node_modules');
  process.chdir(rootDir);

  // Symlinks to packages need to reach back to the workspace/runfiles directory
  const workspaceRelative = path.relative('.', workspaceDir);
  const runfilesRelative = runfiles.dir ? path.relative('.', runfiles.dir) : undefined;

  // Now add symlinks to each of our first-party packages so they appear under the node_modules tree
  const links = [];

  const linkModule =
      async (name: string, modulePath: string) => {
    let target: string|undefined;

    // Look in the runfiles first
    // TODO: this could be a method in the Runfiles class
    if (runfiles.manifest) {
      target = runfiles.lookupDirectory(modulePath);
    } else if (runfilesRelative) {
      target = path.join(runfilesRelative, modulePath);
    }

    // It sucks that we have to do a FS call here.
    // TODO: could we know which packages are statically linked??
    if (!target || !await exists(target)) {
      // Try the execroot
      target = path.join(workspaceRelative, toWorkspaceDir(modulePath));
    }

    await symlink(target, name);
  }

  for (const m of Object.keys(modules)) {
    links.push(linkModule(m, modules[m]));
  }

  await Promise.all(links);

  return 0;
}

if (require.main === module) {
  (async () => {
    process.exitCode = await main(process.argv.slice(2), new Runfiles());
  })();
}