runner.js - Stryker report

/**
 * @license Copyright 2016 Google Inc. All Rights Reserved.
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 */
'use strict';

const isDeepEqual = require(0'lodash.isequal');
const Driver = require(1'./gather/driver.js');
const GatherRunner = require(2'./gather/gather-runner');
const ReportScoring = require(3'./scoring');
const Audit = require(4'./audits/audit');
const log = require(5'lighthouse-logger');
const i18n = require(6'./lib/i18n/i18n.js');
const assetSaver = require(7'./lib/asset-saver');
const fs = require(8'fs');
const path = require(9'path');
const URL = require(10'./lib/url-shim');
const Sentry = require(11'./lib/sentry');
const generateReport = require(12'./report/report-generator').generateReport;
const LHError = require(13'./lib/lh-error.js');

/** @typedef {import('./gather/connections/connection.js')} Connection */
/** @typedef {import('./config/config.js')} Config */

class Runner {
  /**
   * @param {Connection} connection
   * @param {{config: Config, url?: string, driverMock?: Driver}} runOpts
   * @return {Promise<LH.RunnerResult|undefined>}
   */
  static async run(connection, runOpts) 14{
    const settings = runOpts.config.settings;
    try 15{
      const runnerStatus = 16{msg: 17'Runner setup', id: 18'lh:runner:run'};
      log.time(runnerStatus, 19'verbose');

      /**
       * List of top-level warnings for this Lighthouse run.
       * @type {Array<string>}
       */
      const lighthouseRunWarnings = [];

      const sentryContext = Sentry.getContext();
      Sentry.captureBreadcrumb(20{
        message: 21'Run started',
        category: 22'lifecycle',
        data: 232425sentryContext && sentryContext.extra,
      });

      // User can run -G solo, -A solo, or -GA together
      // -G and -A will run partial lighthouse pipelines,
      // and -GA will run everything plus save artifacts to disk

      // Gather phase
      // Either load saved artifacts off disk or from the browser
      let artifacts;
      let requestedUrl;
      if (262728settings.auditMode && 29!settings.gatherMode) 30{
        // No browser required, just load the artifacts from disk.
        const path = Runner._getArtifactsPath(settings);
        artifacts = await assetSaver.loadArtifacts(path);
        requestedUrl = artifacts.URL.requestedUrl;

        if (313233!requestedUrl) 34{
          throw new Error(35'Cannot run audit mode on empty URL');
        }
        if (363738runOpts.url && 39!URL.equalWithExcludedFragments(runOpts.url, requestedUrl)) 40{
          throw new Error(41'Cannot run audit mode on different URL');
        }
      } else 42{
        if (434445464748typeof runOpts.url !== 49'string' || 505152runOpts.url.length === 0) 53{
          throw new Error(54`You must provide a url to the runner. '${runOpts.url}' provided.`);
        }

        try 55{
          // Use canonicalized URL (with trailing slashes and such)
          requestedUrl = new URL(runOpts.url).href;
        } catch (e) 56{
          throw new Error(57'The url provided should have a proper protocol and hostname.');
        }

        artifacts = await Runner._gatherArtifactsFromBrowser(requestedUrl, runOpts, connection);
        // -G means save these to ./latest-run, etc.
        if (5859settings.gatherMode) 60{
          const path = Runner._getArtifactsPath(settings);
          await assetSaver.saveArtifacts(artifacts, path);
        }
      }

      // Potentially quit early
      if (616263settings.gatherMode && 64!settings.auditMode) return;

      // Audit phase
      if (656667!runOpts.config.audits) 68{
        throw new Error(69'No audits to evaluate.');
      }
      const auditResults = await Runner._runAudits(settings, runOpts.config.audits, artifacts,
          lighthouseRunWarnings);

      // LHR construction phase
      const resultsStatus = 70{msg: 71'Generating results...', id: 72'lh:runner:generate'};
      log.time(resultsStatus);

      if (7374artifacts.LighthouseRunWarnings) 75{
        lighthouseRunWarnings.push(...artifacts.LighthouseRunWarnings);
      }

      // Entering: conclusion of the lighthouse result object
      const lighthouseVersion = require(76'../package.json').version;

      /** @type {Object<string, LH.Audit.Result>} */
      const resultsById = {};
      for (const audit of auditResults) 77{
        resultsById[audit.id] = audit;
      }

      /** @type {Object<string, LH.Result.Category>} */
      let categories = {};
      if (7879runOpts.config.categories) 80{
        categories = ReportScoring.scoreAllCategories(runOpts.config.categories, resultsById);
      }

      log.timeEnd(resultsStatus);
      log.timeEnd(runnerStatus);

      /** @type {LH.Result} */
      const lhr = 81{
        userAgent: artifacts.HostUserAgent,
        environment: 82{
          networkUserAgent: artifacts.NetworkUserAgent,
          hostUserAgent: artifacts.HostUserAgent,
          benchmarkIndex: artifacts.BenchmarkIndex,
        },
        lighthouseVersion,
        fetchTime: artifacts.fetchTime,
        requestedUrl: requestedUrl,
        finalUrl: artifacts.URL.finalUrl,
        runWarnings: lighthouseRunWarnings,
        runtimeError: Runner.getArtifactRuntimeError(artifacts),
        audits: resultsById,
        configSettings: settings,
        categories,
        categoryGroups: 838485runOpts.config.groups || undefined,
        timing: this._getTiming(artifacts),
        i18n: 86{
          rendererFormattedStrings: i18n.getRendererFormattedStrings(settings.locale),
          icuMessagePaths: {},
        },
      };

      // Replace ICU message references with localized strings; save replaced paths in lhr.
      lhr.i18n.icuMessagePaths = i18n.replaceIcuMessageInstanceIds(lhr, settings.locale);

      // Create the HTML, JSON, and/or CSV string
      const report = generateReport(lhr, settings.output);

      return 87{lhr, artifacts, report};
    } catch (err) 88{
      // i18n error strings
      err.friendlyMessage = i18n.getFormatted(err.friendlyMessage, settings.locale);
      await Sentry.captureException(err, 89{level: 90'fatal'});
      throw err;
    }
  }

  /**
   * This handles both the auditMode case where gatherer entries need to be merged in and
   * the gather/audit case where timingEntriesFromRunner contains all entries from this run,
   * including those also in timingEntriesFromArtifacts.
   * @param {LH.Artifacts} artifacts
   * @return {LH.Result.Timing}
   */
  static _getTiming(artifacts) 91{
    const timingEntriesFromArtifacts = 929394artifacts.Timing || [];
    const timingEntriesFromRunner = log.takeTimeEntries();
    const timingEntriesKeyValues = 95[
      ...timingEntriesFromArtifacts,
      ...timingEntriesFromRunner,
      // As entries can share a name, dedupe based on the startTime timestamp
    ].map(entry => /** @type {[number, PerformanceEntry]} */ (96[entry.startTime, entry]));
    const timingEntries = Array.from(new Map(timingEntriesKeyValues).values());
    const runnerEntry = timingEntries.find(e => 979899e.name === 100'lh:runner:run');
    return 101{entries: timingEntries, total: 102103104105106107runnerEntry && runnerEntry.duration || 0};
  }

  /**
   * Establish connection, load page and collect all required artifacts
   * @param {string} requestedUrl
   * @param {{config: Config, driverMock?: Driver}} runnerOpts
   * @param {Connection} connection
   * @return {Promise<LH.Artifacts>}
   */
  static async _gatherArtifactsFromBrowser(requestedUrl, runnerOpts, connection) 108{
    if (109110111!runnerOpts.config.passes) 112{
      throw new Error(113'No browser artifacts are either provided or requested.');
    }
    const driver = 114115116runnerOpts.driverMock || new Driver(connection);
    const gatherOpts = 117{
      driver,
      requestedUrl,
      settings: runnerOpts.config.settings,
    };
    const artifacts = await GatherRunner.run(runnerOpts.config.passes, gatherOpts);
    return artifacts;
  }

  /**
   * Run all audits with specified settings and artifacts.
   * @param {LH.Config.Settings} settings
   * @param {Array<LH.Config.AuditDefn>} audits
   * @param {LH.Artifacts} artifacts
   * @param {Array<string>} runWarnings
   * @return {Promise<Array<LH.Audit.Result>>}
   */
  static async _runAudits(settings, audits, artifacts, runWarnings) 118{
    const status = 119{msg: 120'Analyzing and running audits...', id: 121'lh:runner:auditing'};
    log.time(status);

    if (122123artifacts.settings) 124{
      const overrides = 125{
        locale: undefined,
        gatherMode: undefined,
        auditMode: undefined,
        output: undefined,
      };
      const normalizedGatherSettings = Object.assign({}, artifacts.settings, overrides);
      const normalizedAuditSettings = Object.assign({}, settings, overrides);

      if (126127128!isDeepEqual(normalizedGatherSettings, normalizedAuditSettings)) 129{
        throw new Error(130'Cannot change settings between gathering and auditing');
      }
    }

    // Members of LH.Audit.Context that are shared across all audits.
    const sharedAuditContext = 131{
      settings,
      LighthouseRunWarnings: runWarnings,
      computedCache: new Map(),
    };

    // Run each audit sequentially
    const auditResults = [];
    for (const auditDefn of audits) 132{
      const auditResult = await Runner._runAudit(auditDefn, artifacts, sharedAuditContext);
      auditResults.push(auditResult);
    }

    log.timeEnd(status);
    return auditResults;
  }

  /**
   * Checks that the audit's required artifacts exist and runs the audit if so.
   * Otherwise returns error audit result.
   * @param {LH.Config.AuditDefn} auditDefn
   * @param {LH.Artifacts} artifacts
   * @param {Pick<LH.Audit.Context, 'settings'|'LighthouseRunWarnings'|'computedCache'>} sharedAuditContext
   * @return {Promise<LH.Audit.Result>}
   * @private
   */
  static async _runAudit(auditDefn, artifacts, sharedAuditContext) 133{
    const audit = auditDefn.implementation;
    const status = 134{
      msg: 135`Evaluating: ${i18n.getFormatted(audit.meta.title, 136'en-US')}`,
      id: 137`lh:audit:${audit.meta.id}`,
    };
    log.time(status);

    let auditResult;
    try 138{
      // Return an early error if an artifact required for the audit is missing or an error.
      for (const artifactName of audit.meta.requiredArtifacts) 139{
        const noArtifact = 140141142artifacts[artifactName] === undefined;

        // If trace required, check that DEFAULT_PASS trace exists.
        // TODO: need pass-specific check of networkRecords and traces.
        const noTrace = 143144145146147148artifactName === 149'traces' && 150!artifacts.traces[Audit.DEFAULT_PASS];

        if (151152153noArtifact || noTrace) 154{
          log.warn(155'Runner',
              156`${artifactName} gatherer, required by audit ${audit.meta.id}, did not run.`);
          throw new Error(157`Required ${artifactName} gatherer did not run.`);
        }

        // If artifact was an error, output error result on behalf of audit.
        if (158159artifacts[artifactName] instanceof Error) 160{
          /** @type {Error} */
          // @ts-ignore An artifact *could* be an Error, but caught here, so ignore elsewhere.
          const artifactError = artifacts[artifactName];

          Sentry.captureException(artifactError, 161{
            tags: 162{gatherer: artifactName},
            level: 163'error',
          });

          log.warn(164'Runner', 165166`${artifactName} gatherer, required by audit ${audit.meta.id},` +
            167` encountered an error: ${artifactError.message}`);

          // Create a friendlier display error and mark it as expected to avoid duplicates in Sentry
          const error = new Error(
              168`Required ${artifactName} gatherer encountered an error: ${artifactError.message}`);
          // @ts-ignore Non-standard property added to Error
          error.expected = 169true;
          throw error;
        }
      }

      // all required artifacts are in good shape, so we proceed
      const auditOptions = Object.assign({}, audit.defaultOptions, auditDefn.options);
      const auditContext = 170{
        options: auditOptions,
        ...sharedAuditContext,
      };

      const product = await audit.audit(artifacts, auditContext);
      auditResult = Audit.generateAuditResult(audit, product);
    } catch (err) 171{
      log.warn(audit.meta.id, 172`Caught exception: ${err.message}`);

      Sentry.captureException(err, 173{tags: 174{audit: audit.meta.id}, level: 175'error'});
      // Errors become error audit result.
      const errorMessage = err.friendlyMessage ? err.friendlyMessage : err.message;
      auditResult = Audit.generateErrorAuditResult(audit, errorMessage);
    }

    log.timeEnd(status);
    return auditResult;
  }

  /**
   * Returns first runtimeError found in artifacts.
   * @param {LH.Artifacts} artifacts
   * @return {LH.Result['runtimeError']|undefined}
   */
  static getArtifactRuntimeError(artifacts) 176{
    for (const possibleErrorArtifact of Object.values(artifacts)) 177{
      if (178179180possibleErrorArtifact instanceof LHError && possibleErrorArtifact.lhrRuntimeError) 181{
        const errorMessage = 182183184possibleErrorArtifact.friendlyMessage || possibleErrorArtifact.message;

        return 185{
          code: possibleErrorArtifact.code,
          message: errorMessage,
        };
      }
    }

    return undefined;
  }

  /**
   * Returns list of audit names for external querying.
   * @return {Array<string>}
   */
  static getAuditList() 186{
    const ignoredFiles = 187[
      188'audit.js',
      189'violation-audit.js',
      190'accessibility/axe-audit.js',
      191'multi-check-audit.js',
      192'byte-efficiency/byte-efficiency-audit.js',
      193'manual/manual-audit.js',
    ];

    const fileList = 194[
      ...fs.readdirSync(path.join(__dirname, 195'./audits')),
      ...fs.readdirSync(path.join(__dirname, 196'./audits/dobetterweb')).map(f => 197`dobetterweb/${f}`),
      ...fs.readdirSync(path.join(__dirname, 198'./audits/metrics')).map(f => 199`metrics/${f}`),
      ...fs.readdirSync(path.join(__dirname, 200'./audits/seo')).map(f => 201`seo/${f}`),
      ...fs.readdirSync(path.join(__dirname, 202'./audits/seo/manual')).map(f => 203`seo/manual/${f}`),
      ...fs.readdirSync(path.join(__dirname, 204'./audits/accessibility'))
          .map(f => 205`accessibility/${f}`),
      ...fs.readdirSync(path.join(__dirname, 206'./audits/accessibility/manual'))
          .map(f => 207`accessibility/manual/${f}`),
      ...fs.readdirSync(path.join(__dirname, 208'./audits/byte-efficiency'))
          .map(f => 209`byte-efficiency/${f}`),
      ...fs.readdirSync(path.join(__dirname, 210'./audits/manual')).map(f => 211`manual/${f}`),
    ];
    return fileList.filter(f => 212{
      return 213214215/\.js$/.test(f) && 216!ignoredFiles.includes(f);
    }).sort();
  }

  /**
   * Returns list of gatherer names for external querying.
   * @return {Array<string>}
   */
  static getGathererList() 217{
    const fileList = 218[
      ...fs.readdirSync(path.join(__dirname, 219'./gather/gatherers')),
      ...fs.readdirSync(path.join(__dirname, 220'./gather/gatherers/seo')).map(f => 221`seo/${f}`),
      ...fs.readdirSync(path.join(__dirname, 222'./gather/gatherers/dobetterweb'))
          .map(f => 223`dobetterweb/${f}`),
    ];
    return fileList.filter(f => 224225226/\.js$/.test(f) && 227228229f !== 230'gatherer.js').sort();
  }

  /**
   * Get path to use for -G and -A modes. Defaults to $CWD/latest-run
   * @param {LH.Config.Settings} settings
   * @return {string}
   */
  static _getArtifactsPath(settings) 231{
    const {auditMode, gatherMode} = settings;

    // This enables usage like: -GA=./custom-folder
    if (232233234typeof auditMode === 235'string') return path.resolve(process.cwd(), auditMode);
    if (236237238typeof gatherMode === 239'string') return path.resolve(process.cwd(), gatherMode);

    return path.join(process.cwd(), 240'latest-run');
  }
}

module.exports = Runner;