webpack#Stats TypeScript Examples

/**
 * Find the original module that required the transient dependency. Returns
 * undefined if the module is a first level dependency.
 * @param {Object} issuer - Module issuer
 */
// eslint-disable-next-line no-null/no-null
function findExternalOrigin(stats: Stats, issuer: Module | null): any {
    if (!isNil(issuer) && (issuer as any).rawRequest.startsWith('./')) {
        return findExternalOrigin(stats, stats.compilation.moduleGraph.getIssuer(issuer));
    }
    return issuer;
}

function getExternalModules(stats: Stats | undefined): any[] {
    if (!stats?.compilation.chunks) {
        return [];
    }
    const externals = new Set();
    for (const chunk of stats.compilation.chunks) {
        const modules = stats.compilation.chunkGraph.getChunkModules(chunk);
        if (!modules) {
            continue;
        }

        // Explore each module within the chunk (built inputs):
        for (const module of modules) {
            if (isExternalModule(module)) {
                externals.add({
                    origin: get(findExternalOrigin(stats, stats.compilation.moduleGraph.getIssuer(module)), 'rawRequest'),
                    external: getExternalModuleName(module as ExternalModule)
                });
            }
        }
    }
    return Array.from(externals);
}

runBuild = (configuration: Configuration[]): Promise<Stats> => {
    const executor = (resolve: (value: Stats) => void) => webpack(
        configuration as Configuration, // https://github.com/Microsoft/TypeScript/issues/14107
        (err?: Error, stats?: Stats) => {
            if (err) {
                logger.error(err.toString());
                process.exit(22);
            }

            if (!stats) {
                logger.error('Unknown error: webpack does not return its build stats');
                process.exit(22);
            }

            const toJsonOptions = {all: false, errors: true, warnings: true, assets: true};
            // webpack的`toJson`的定义是错的
            const {errors, warnings} = stats.toJson(toJsonOptions);
            for (const error of reject(isNil, errors ?? [])) {
                printWebpackResult('error', error as unknown as WebpackResult);
            }
            for (const warning of reject(isNil, warnings ?? [])) {
                printWebpackResult('warn', warning as unknown as WebpackResult);
            }

            if (stats.hasErrors()) {
                process.exit(22);
            }

            resolve(stats);
        }
    );

    return new Promise(executor);
}

export default async function runExample(
  example: string,
): Promise<{
  stats: Stats
  errors: string[]
  warnings: string[]
  assets: Record<string, string>
}> {
  const exampleConfig = require(path.resolve(__dirname, '../../examples', example, 'webpack.config.js'))
  const compiler = webpackTestUtils.getCompiler(exampleConfig)

  const stats = await webpackTestUtils.compile(compiler)
  const errors = webpackTestUtils.getErrors(stats)
  const warnings = webpackTestUtils.getWarnings(stats)
  const assets = webpackTestUtils.readAssets(compiler, stats)

  return {
    stats,
    errors,
    warnings,
    assets,
  }
}

drawBuildReport = (stats: Stats[]): void => {
    const info = stats.flatMap(extractBuildInfo);
    const initialChunks = new Set(info.flatMap(v => v.initialChunks));
    const toAsset = (value: WebpackCompileAsset): Asset => {
        return {
            name: value.name,
            size: value.size,
            initial: initialChunks.has(value.name),
        };
    };
    const assets = info.flatMap(v => v.assets.map(toAsset));
    drawAssetReport(assets);
}

export function readAssets(compiler: Compiler, stats: Stats): Record<string, string> {
  const assets: Record<string, string> = {}

  Object.keys(stats.compilation.assets).forEach(asset => {
    assets[asset] = readAsset(asset, compiler, stats)
  })

  return assets
}

export function readAsset(asset: string, compiler: Compiler, stats: Stats): string {
  const outFs = (compiler.outputFileSystem as any) as IFs
  const outputPath: string = stats.compilation.outputOptions.path

  let data = ''
  let targetFile = asset

  const queryStringIdx = targetFile.indexOf('?')

  if (queryStringIdx >= 0) {
    targetFile = targetFile.substr(0, queryStringIdx)
  }

  try {
    data = outFs.readFileSync(path.join(outputPath, targetFile)).toString()
  } catch (error) {
    data = error.toString()
  }

  return data
}

export function getWarnings(stats: Stats): string[] {
  return normalizeErrors(stats.compilation.warnings)
}

export default async function compile(
  fixture: string,
  loaderOptions: Options,
  sourceMap = false,
): Promise<{
  stats: Stats
  errors: string[]
  warnings: string[]
  exports: any
}> {
  const compiler = webpackTestUtils.getCompiler({
    context: path.resolve(__dirname, '../fixtures', fixture),
    entry: path.resolve(__dirname, '../fixtures', fixture, 'entry.js'),
    devtool: sourceMap ? 'source-map' : undefined,
    module: {
      rules: [
        {
          test: /\.wxss$/,
          loader: require.resolve('@mpflow/wxss-loader'),
          options: loaderOptions,
        },
        {
          test: /\.less$/,
          use: [
            {
              loader: require.resolve('@mpflow/wxss-loader'),
              options: loaderOptions,
            },
            {
              loader: require.resolve('less-loader'),
            },
          ],
        },
        {
          test: /\.(png|jpg|gif|svg|eot|ttf|woff|woff2)$/i,
          loader: 'file-loader',
          options: { name: '[name].[hash:8].[ext]' },
        },
      ],
    },
    optimization: {
      namedModules: true,
    },
  })

  const stats = await webpackTestUtils.compile(compiler)
  const errors = webpackTestUtils.getErrors(stats)
  const warnings = webpackTestUtils.getWarnings(stats)
  const exports = webpackTestUtils.getExecutedCode('main.bundle.js', compiler, stats)

  return {
    stats,
    errors,
    warnings,
    exports,
  }
}

export default async function compile(
  fixture: string,
  loaderOptions: Options,
): Promise<{
  stats: Stats
  errors: string[]
  warnings: string[]
  exports: any
}> {
  const compiler = webpackTestUtils.getCompiler({
    context: path.resolve(__dirname, '../fixtures', fixture),
    entry: path.resolve(__dirname, '../fixtures', fixture, 'entry.js'),
    resolve: {
      alias: {
        '/img.png': path.resolve(__dirname, '../fixtures', fixture, 'img.png'),
      },
    },
    module: {
      rules: [
        {
          test: /\.wxml$/,
          loader: require.resolve('@mpflow/wxml-loader'),
          options: loaderOptions,
        },
        {
          test: /\.wxs$/,
          loader: 'file-loader',
          options: { name: '[name].[hash:8].[ext]' },
        },
        {
          test: /\.(png|jpg|gif|svg|eot|ttf|woff|woff2)$/i,
          loader: 'file-loader',
          options: { name: '[name].[hash:8].[ext]' },
        },
      ],
    },
  })

  const stats = await webpackTestUtils.compile(compiler)
  const errors = webpackTestUtils.getErrors(stats)
  const warnings = webpackTestUtils.getWarnings(stats)
  const exports = webpackTestUtils.getExecutedCode('main.bundle.js', compiler, stats)

  return {
    stats,
    errors,
    warnings,
    exports,
  }
}

export function getExecutedCode<T = any>(asset: string, compiler: Compiler, stats: Stats): T {
  return execute<T>(readAsset(asset, compiler, stats))
}

export function getModuleSource(id: string, stats: Stats): string | undefined {
  const { modules } = stats.toJson({ source: true })
  const module = modules?.find(module => module.name.endsWith(id))
  return module?.source
}

export function getErrors(stats: Stats): string[] {
  return normalizeErrors(stats.compilation.errors)
}

export function compile(compiler: Compiler): Promise<Stats> {
  return new Promise((resolve, reject) => {
    compiler.run((error, stats) => (error ? reject(error) : resolve(stats)))
  })
}

extractBuildInfo = (stats: Stats) => {
    const {children = []} = stats.toJson('normal');
    const entrypoints = children.flatMap(child => Object.values(child?.entrypoints ?? {}));
    const initialChunks = entrypoints.flatMap(entry => entry.chunks);
    const assets = children.map(child => child.assets ?? []).flatMap(v => v) as WebpackCompileAsset[];
    return {initialChunks, assets};
}

describe('ReactLooselyLazyPlugin', () => {
  type TestWebpackPluginOptions = {
    publicPath?: string;
  };

  const testWebpackPlugin = async ({
    publicPath,
  }: TestWebpackPluginOptions = {}) => {
    const rootPath = join(__dirname, '..', '..', '..', '..');
    const projectRoot = join(__dirname, 'app');
    const manifestFilename = 'manifest.json';

    const config = {
      devtool: 'source-map' as const,
      entry: {
        main: join(projectRoot, 'src', 'index.tsx'),
      },
      mode: 'production' as const,
      output: {
        path: join(projectRoot, 'dist'),
        filename: '[name].js',
        publicPath: '/dist/',
      },
      module: {
        rules: [
          {
            test: /\.tsx?$/,
            use: {
              loader: 'babel-loader',
              options: {
                babelrc: false,
                presets: [['@babel/preset-env', { modules: false }]],
              },
            },
          },
          {
            test: /\.css$/,
            use: [MiniCssExtractPlugin.loader, 'css-loader'],
          },
        ],
      },
      plugins: [
        new ReactLooselyLazyPlugin({
          filename: manifestFilename,
          publicPath,
        }),
        // @ts-expected-error New types do not work with webpack 4
        new MiniCssExtractPlugin() as any,
      ],
      resolve: {
        alias: {
          'custom-alias': join(projectRoot, 'lib', 'custom-alias.tsx'),
        },
        extensions: ['.tsx', '.ts', '.js'],
        plugins: [new TsconfigPathsPlugin()],
      },
    };

    type WebpackOutput = {
      manifest: Manifest;
      stats: Stats.ToJsonOutput;
    };

    const result = await new Promise<WebpackOutput>((resolve, reject) => {
      webpack(config, (err, stats) => {
        if (err) reject(err);

        const asset = stats.compilation.assets[manifestFilename];
        resolve({
          manifest: asset ? JSON.parse(asset.source()) : undefined,
          stats: stats.toJson(),
        });
      });
    });

    const { manifest, stats } = result;

    expect(stats).toMatchObject({
      errors: [],
      warnings: [],
    });

    const integrationAppPath = './packages/testing/integration-app';
    const projectRootPath = `./${relative(rootPath, projectRoot)}`;

    expect(manifest).toEqual({
      publicPath: publicPath ?? '/dist/',
      assets: {
        [`${integrationAppPath}/src/ui/concatenated-module/index.tsx`]: [
          'async-concatenated-module.js',
        ],
        [`${integrationAppPath}/src/ui/external-assets/index.tsx`]: [
          'async-external-assets.css',
          'async-external-assets.js',
        ],
        [`${integrationAppPath}/src/ui/lazy-after-paint.tsx`]: [
          'async-lazy-after-paint.js',
        ],
        [`${integrationAppPath}/src/ui/lazy-for-paint.tsx`]: [
          'async-lazy-for-paint.js',
        ],
        [`${integrationAppPath}/src/ui/lazy.tsx`]: ['async-lazy.js'],
        [`${integrationAppPath}/src/ui/multiple-usages.tsx`]: [
          'async-multiple-usages-one.js',
        ],
        [`${integrationAppPath}/src/ui/named-lazy-for-paint.tsx`]: [
          'async-named-lazy-for-paint.js',
        ],
        [`${integrationAppPath}/src/ui/nested-lazy/index.tsx`]: [
          'async-nested-lazy.js',
        ],
        [`${integrationAppPath}/src/ui/nested-lazy/main.tsx`]: [
          'async-inner-nested-lazy.js',
        ],
        [`${integrationAppPath}/src/ui/typed-lazy-for-paint.tsx`]: [
          'async-typed-lazy-for-paint.js',
        ],
        [`${projectRootPath}/lib/custom-alias.tsx`]: ['async-custom-alias.js'],
      },
    });
  };

  describe('creates the manifest', () => {
    it('using the default options', async () => {
      await testWebpackPlugin();
    });

    it('with an overridden publicPath when it is provided', async () => {
      await testWebpackPlugin({
        publicPath: 'https://cdn.example.com/',
      });
    });
  });
});

/**
 * We need a performant algorithm to install the packages for each single
 * function (in case we package individually).
 * (1) We fetch ALL packages needed by ALL functions in a first step
 * and use this as a base npm checkout. The checkout will be done to a
 * separate temporary directory with a package.json that contains everything.
 * (2) For each single compile we copy the whole node_modules to the compile
 * directory and create a (function) compile specific package.json and store
 * it in the compile directory. Now we start npm again there, and npm will just
 * remove the superfluous packages and optimize the remaining dependencies.
 * This will utilize the npm cache at its best and give us the needed results
 * and performance.
 */
export async function packExternalModules(context: ConfigurationContext, stats: Stats | undefined): Promise<void> {
    const verbose = false;
    const { cfg, pkg, runtime } = context;
    const config = ConfigUtil.getMalaguConfig(cfg, BACKEND_TARGET);
    const configuration = ConfigurationContext.getConfiguration(BACKEND_TARGET, context.configurations);
    const includes = config.includeModules;
    const packagerOptions = { nonInteractive: true, ignoreOptional: true, ...config.packagerOptions };
    const scripts: any[] = packagerOptions.scripts || [];

    if (isEmpty(includes) && includes !== true || !configuration) {
        return;
    }

    const outputPath = configuration.output.get('path');

    // Read plugin configuration
    const packageForceIncludes = includes.forceInclude || [];
    const packageForceExcludes = includes.forceExclude || [];
    const packageForceIncludeAll = includes.forceIncludeAll;
    const packagePath = includes.packagePath && join(process.cwd(), includes.packagePath) || join(process.cwd(), 'package.json');
    const packageScripts = scripts.reduce((accumulator, script, index) => {
        accumulator[`script${index}`] = script;
        return accumulator;
    },
        {}
    );

    const packager = getPackager(context.cfg.rootConfig.packager, process.cwd());

    const sectionNames = packager.copyPackageSectionNames;
    const packageJson = await readJSON(packagePath);
    if (packageForceIncludeAll) {
        for (const d of Object.keys(packageJson.dependencies)) {
            if (!packageForceIncludes.includes(d)) {
                packageForceIncludes.push(d);
            }
        }
    }
    const packageSections = pick(packageJson, sectionNames);
    if (!isEmpty(packageSections)) {
        console.log(`Using package.json sections ${Object.keys(packageSections).join(', ')}`);
    }

    const dependencyGraph = await packager.getProdDependencies(1);

    const problems = dependencyGraph.problems || [];
    if (verbose && !isEmpty(problems)) {
        console.log(`Ignoring ${problems.length} NPM errors:`);
        problems.forEach((problem: any) => {
            console.log(`=> ${problem}`);
        });
    }

    // (1) Generate dependency composition
    const externalModules = getExternalModules(stats).concat(packageForceIncludes.map((whitelistedPackage: string) => ({
        external: whitelistedPackage
    })));
    const compositeModules = uniq(getProdModules(uniq(externalModules), packagePath, dependencyGraph, packageForceExcludes, runtime));
    removeExcludedModules(compositeModules, packageForceExcludes, true);

    if (isEmpty(compositeModules)) {
        // The compiled code does not reference any external modules at all
        console.log('No external modules needed');
        return;
    }

    // (1.a) Install all needed modules
    const compositeModulePath = outputPath;
    const compositePackageJson = join(compositeModulePath, 'package.json');

    // (1.a.1) Create a package.json
    const compositePackage = defaults(
        {
            name: pkg.pkg.name,
            version: pkg.pkg.version,
            description: `Packaged externals for ${pkg.pkg.name}`,
            private: true,
            scripts: packageScripts
        },
        packageSections
    );
    const relPath = relative(compositeModulePath, dirname(packagePath));
    addModulesToPackageJson(compositeModules, compositePackage, relPath);
    writeJSONSync(compositePackageJson, compositePackage, { spaces: 2 });

    // (1.a.2) Copy package-lock.json if it exists, to prevent unwanted upgrades
    const packageLockPath = join(dirname(packagePath), packager.lockfileName);
    const hasPackageLock = await pathExists(packageLockPath);
    if (hasPackageLock) {
        console.log('?  malagu package lock found - Using locked versions');
        try {
            let packageLockFile = await packager.readLockfile(packageLockPath);
            packageLockFile = packager.rebaseLockfile(relPath, packageLockFile);
            packager.writeLockfile(join(compositeModulePath, packager.lockfileName), packageLockFile);
        } catch (err) {
            console.warn(`Warning: Could not read lock file: ${err.message}`);
        }
    }

    const start = now();
    for (const compositeModule of compositeModules) {
        console.log(`?  malagu external modules - ${compositeModule}`);
    }
    await packager.install(packagerOptions, compositeModulePath);
    if (verbose) {
        console.log(`Package took [${now() - start} ms]`);
    }

    // Prune extraneous packages - removes not needed ones
    const startPrune = now();
    await packager.prune(packagerOptions, compositeModulePath);
    if (verbose) {
        console.log(`Prune: ${compositeModulePath} [${now() - startPrune} ms]`);
    }

    // Prune extraneous packages - removes not needed ones
    const startRunScripts = now();
    await packager.runScripts(Object.keys(packageScripts), compositeModulePath);
    if (verbose) {
        console.log(`Run scripts: ${compositeModulePath} [${now() - startRunScripts} ms]`);
    }
}

// create the node vm
async function createVm(
  pluginPath: string,
  { builtin = ['*'], external = true, isTypeScript = false } = {}
): Promise<[boolean, string]> {
  let pluginCode: string;

  if (isTypeScript) {
    try {
      const tsBuildPath = path.join(pluginPath, '.build');
      const sourceFileName = path.join(pluginPath, MAIN_FILE_TS);
      const outputPath = path.join(tsBuildPath, 'plugin.js');
      mkdir(tsBuildPath);
      mkdir(path.join(tsBuildPath, '.cache'));

      const stats = await new Promise<Stats>((resolve, reject) => {
        webpack(
          Object.freeze({
            target: 'node',
            context: __dirname,
            mode: 'development',
            entry: sourceFileName,
            devtool: 'source-map',
            output: {
              iife: false,
              library: {
                type: 'commonjs',
              },
              path: tsBuildPath,
              filename: 'plugin.js',
            },
            resolve: {
              extensions: ['.ts', '.js', '.json'],
              alias: {
                // src is the only hard coded path
                src: path.resolve(pluginPath, 'src'),
              },
            },
            cache: {
              type: 'filesystem',
              cacheLocation: path.join(tsBuildPath, '.cache'),
              allowCollectingMemory: false,
              idleTimeout: 0,
              idleTimeoutForInitialStore: 0,
              // cache age is one week. plugins probably do not need
              // to be rebuilt every day
              maxAge: 1000 * 60 * 60 * 24 * 7,
              profile: true,
            },
            module: {
              rules: [
                {
                  test: /\.[jt]s$/,
                  exclude: /(node_modules)/,
                  use: {
                    loader: 'swc-loader',
                    options: {
                      sourceMaps: true,
                      cwd: pluginPath,
                      isModule: true,
                      jsc: {
                        target: 'es2020',
                        parser: {
                          syntax: 'typescript',
                        },
                        transform: {},
                      },

                      module: {
                        type: 'commonjs',
                        strictMode: false,
                      },
                    },
                  },
                },
              ],
            },
          }),
          (err, stats) => (err ? reject(err) : resolve(stats))
        );
      });

      if (stats.hasErrors()) {
        for (const err of stats.toJson().errors) {
          Logger.errorp(err.moduleName, err.file);
          Logger.errorp(err.message);
        }
      }

      if (stats.hasWarnings()) {
        for (const warning of stats.toJson().warnings) {
          Logger.warnp(warning.moduleName, warning.file);
          Logger.warnp(warning.message);
        }
      }

      pluginCode = fs.readFileSync(outputPath).toString();
    } catch (err) {
      Logger.errorp(pluginName.brightRed, err);
      return [false, 'failed compiling building typescript'];
    }

    // update omegga.d.ts to latest on plugin compile
    try {
      const gitIgnore = path.join(pluginPath, '.gitignore');
      const omeggaTypesDst = path.join(pluginPath, 'omegga.d.ts');
      const omeggaTypesSrc = path.join(
        __dirname,
        '../../../../templates/safe-ts/omegga.d.ts'
      );

      // plugin has gitignore with "omegga.d.ts" in it and omegga has omegga.d.ts
      if (fs.existsSync(gitIgnore) && fs.existsSync(omeggaTypesSrc)) {
        // and the gitignore covers the omegga.d.ts
        const hasOmeggaTypesIgnored = fs
          .readFileSync(gitIgnore)
          .toString()
          .match(/(\.\/)?omegga\.d\.ts/);

        // compare last modified times to avoid unnecessary copies
        const srcLastModified = fs.statSync(omeggaTypesSrc).mtime.getTime();
        const dstLastModified = fs.existsSync(omeggaTypesDst)
          ? fs.statSync(omeggaTypesDst).mtime.getTime()
          : null;
        if (
          hasOmeggaTypesIgnored &&
          (!dstLastModified || srcLastModified > dstLastModified)
        ) {
          fs.copyFileSync(omeggaTypesSrc, omeggaTypesDst);
        }
      }
    } catch (err) {
      Logger.errorp(
        pluginName.brightRed,
        'error copying latest omegga.d.ts to typescript plugin',
        err
      );
    }
  }
  if (vm !== undefined) return [false, 'vm is already created'];

  // create the vm
  vm = new NodeVM({
    console: 'redirect',
    sandbox: {},
    require: {
      external,
      builtin,
      root: pluginPath,
    },
  });

  // plugin log generator function
  const ezLog =
    (
      logFn: 'log' | 'error' | 'info' | 'debug' | 'warn' | 'trace',
      name: string,
      symbol: string
    ) =>
    (...args: any[]) =>
      console[logFn](name.underline, symbol, ...args);

  // special formatting for stdout
  vm.on('console.log', ezLog('log', pluginName, '>>'.green));
  vm.on('console.error', ezLog('error', pluginName.brightRed, '!>'.red));
  vm.on('console.info', ezLog('info', pluginName, '#>'.blue));
  vm.on('console.debug', ezLog('debug', pluginName, '?>'.blue));
  vm.on('console.warn', ezLog('warn', pluginName.brightYellow, ':>'.yellow));
  vm.on('console.trace', ezLog('trace', pluginName, 'T>'.grey));

  global.OMEGGA_UTIL = require('../../../util/index.js');
  // pass in util functions
  vm.freeze(global.OMEGGA_UTIL, 'OMEGGA_UTIL');
  vm.freeze(omegga, 'Omegga');
  vm.freeze(Player, 'Player');

  const file = path.join(pluginPath, MAIN_FILE);
  if (!isTypeScript) {
    try {
      pluginCode = fs.readFileSync(file).toString();
    } catch (e) {
      emit(
        'error',
        'failed to read plugin source: ' + e?.stack ?? e.toString()
      );
      throw 'failed to read plugin source: ' + e?.stack ?? e.toString();
    }
  }

  // proxy the plugin out of the vm
  // potential for performance improvement by using VM.script to precompile plugins
  try {
    const pluginOutput = vm.run(pluginCode, file);
    PluginClass = pluginOutput?.default ?? pluginOutput;
  } catch (e) {
    emit('error', 'plugin failed to init');
    Logger.errorp(pluginName.brightRed, e);
    throw 'plugin failed to init: ' + e?.stack ?? e.toString();
  }

  if (
    !PluginClass ||
    typeof PluginClass !== 'function' ||
    typeof PluginClass.prototype !== 'object'
  ) {
    PluginClass = undefined;
    emit('error', 'plugin does not export a class');
    throw 'plugin does not export a class';
  }

  if (typeof PluginClass.prototype.init !== 'function') {
    PluginClass = undefined;
    emit('error', 'plugin is missing init() function');
    throw 'plugin is missing init() function';
  }

  if (typeof PluginClass.prototype.stop !== 'function') {
    PluginClass = undefined;
    emit('error', 'plugin is missing stop() function');
    throw 'plugin is missing stop() function';
  }
}