chore: fetch cache

Grammar edit from none to non-chromium browser(s)

.eslintignore

@@ -13,6 +13,10 @@ __output-dev
 
 docs/_merged*
 *-mdjs-generated.js
+*-converted-md-source.js
+*-converted-md.js
+
+__example_site-for-check-website
 
 # sanity example has a separate backend that is unrelated to Rocket
 # therefore it does not need to follow it code rules

.github/workflows/release.yml

@@ -12,45 +12,29 @@ jobs:
     name: Release
     runs-on: ubuntu-latest
     steps:
+      - uses: google/wireit@setup-github-actions-caching/v1
+
       - name: Checkout Repo
         uses: actions/checkout@master
         with:
           # This makes Actions fetch all Git history so that Changesets can generate changelogs with the correct commits
           fetch-depth: 0
 
-      - name: Setup Node.js 14.x
+      - name: Setup Node.js 18.x
         uses: actions/setup-node@master
         with:
-          node-version: 14.x
+          node-version: 18.x
           registry-url: 'https://registry.npmjs.org'
 
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-
-      - uses: actions/cache@v2
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
       - name: Install Dependencies
-        run: yarn --frozen-lockfile
-
-      - name: Build Packages
-        run: yarn build:packages
-
-      - name: Build Types
-        run: yarn types
+        run: npm ci
 
       - name: Create Release Pull Request or Publish to npm
         id: changesets
         uses: changesets/action@master
         with:
           # This expects you to have a script called release which does a build for your packages and calls changeset publish
-          publish: yarn release
+          publish: npm run release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/verify.yml

@@ -8,8 +8,9 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [18.x]
     steps:
+      - uses: google/wireit@setup-github-actions-caching/v1
       - uses: actions/checkout@v2
 
       - name: Setup Node ${{ matrix.node-version }}
@@ -17,20 +18,8 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-
-      - uses: actions/cache@v2
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
       - name: Install dependencies
-        run: yarn --frozen-lockfile
+        run: npm ci
 
       - name: Install Playwright dependencies
         run: npx playwright install-deps
@@ -38,40 +27,8 @@ jobs:
       - name: Install Playwright
         run: npx playwright install
 
-      - name: Build Packages
-        run: yarn build:packages
-
       - name: Lint
-        run: yarn lint
+        run: npm run lint
 
       - name: Test
-        run: yarn test
-
-  # verify-windows:
-  #   name: Verify windows
-  #   runs-on: windows-latest
-  #   steps:
-  #     - uses: actions/checkout@v2
-
-  #     - name: Setup Node 12.x
-  #       uses: actions/setup-node@v1
-  #       with:
-  #         node-version: 12.x
-
-  #     - name: Get yarn cache directory path
-  #       id: yarn-cache-dir-path
-  #       run: echo "::set-output name=dir::$(yarn cache dir)"
-
-  #     - uses: actions/cache@v2
-  #       id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-  #       with:
-  #         path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-  #         key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-  #         restore-keys: |
-  #           ${{ runner.os }}-yarn-
-
-  #     - name: Install dependencies
-  #       run: yarn --frozen-lockfile
-
-  #     - name: Test
-  #       run: yarn test
+        run: npm run test

.gitignore

@@ -12,21 +12,16 @@ coverage/
 ## npm
 node_modules
 npm-debug.log
-yarn-error.log
 
 ## temp folders
 /.tmp/
 
-## we prefer yarn.lock
-package-lock.json
-## lock files in packages we do not need to save
-packages/*/yarn.lock
-
 ## build output
 dist
 dist-types
 stats.html
 *.tsbuildinfo
+.wireit
 
 # Rocket Search
 rocket-search-index.json
@@ -49,3 +44,4 @@ docs_backup
 
 ## Local playground
 examples/testing
+__example_site-for-check-website

.vscode/settings.json

@@ -8,5 +8,5 @@
     "**/*-mdjs-generated.js": true,
     "**/dist-types": true,
   },
-  "editor.experimental.stickyScroll.enabled": true
+  "editor.stickyScroll.enabled": true
 }

CONTRIBUTING.md

@@ -32,10 +32,10 @@ git checkout -b my-awesome-fix
 
 ## Preparing Your Local Environment for Development
 
-Now that you have cloned the repository, ensure you have [yarn](https://classic.yarnpkg.com/lang/en/) installed, then run the following commands to set up the development environment.
+Now that you have cloned the repository, ensure you have [node](https://nodejs.org/) installed, then run the following commands to set up the development environment.
 
 ```shell
-yarn install
+npm install
 ```
 
 This will download and install all packages needed.
@@ -50,7 +50,7 @@ If you're making cross-package changes, you need to compile the TypeScript code.
 
 ### Running Tests
 
-To run the tests of a package, it's recommended to `cd` into the package directory and then using `yarn test` to run them. This way you're only running tests of that specific package.
+To run the tests of a package, it's recommended to `cd` into the package directory and then using `npm run test` to run them. This way you're only running tests of that specific package.
 
 ### Integration Testing
 
@@ -58,7 +58,7 @@ To see how your changes integrate with everything together you can use the `test
 
 ## Adding New Packages
 
-For all projects the tsconfig/jsconfig configuration files are auto generated. You need to add an entry to the [./workspace-packages.ts](./workspace-packages.ts) to let it generate a config for you. After adding an entry, run `yarn update-package-configs` to generate the files for you.
+For all projects the tsconfig/jsconfig configuration files are auto generated. You need to add an entry to the [./workspace-packages.ts](./workspace-packages.ts) to let it generate a config for you. After adding an entry, run `npm run update-package-configs` to generate the files for you.
 
 ## Creating a Changeset
 
@@ -70,7 +70,7 @@ This documents your intent to release, and allows you to specify a message that
 Run
 
 ```shell
-yarn changeset
+npm run changeset
 ```
 
 And use the menu to select for which packages you need a release, and then select what kind of release. For the release type, we follow [Semantic Versioning](https://semver.org/), so please take a look if you're unfamiliar.

examples/01-hydration-starter/.gitignore

@@ -3,8 +3,6 @@ node_modules/
 
 # logs
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 
 # environment variables
 .env

examples/02-blog-starter/.gitignore

@@ -3,8 +3,6 @@ node_modules/
 
 # logs
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 
 # environment variables
 .env

examples/03-minimal-starter/.gitignore

@@ -3,8 +3,6 @@ node_modules/
 
 # logs
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 
 # environment variables
 .env

examples/04-sanity-minimal-starter/backend/.gitignore

@@ -1,4 +1,2 @@
 /dist
 /node_modules
-
-yarn.lock

examples/04-sanity-minimal-starter/backend/package.json

@@ -3,7 +3,6 @@
   "private": true,
   "version": "1.0.0",
   "description": "",
-  "main": "package.json",
   "author": "Jaydan Urwin <jaydan@jaydanurwin.com>",
   "license": "UNLICENSED",
   "scripts": {

examples/04-sanity-minimal-starter/frontend/.gitignore

@@ -3,8 +3,6 @@ node_modules/
 
 # logs
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 
 # environment variables
 .env

examples/50-landing-theme-spark/.gitignore

@@ -3,8 +3,6 @@ node_modules/
 
 # logs
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 
 # environment variables
 .env

examples/51-docs-theme-launch/.gitignore

@@ -3,8 +3,6 @@ node_modules/
 
 # logs
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 
 # environment variables
 .env

package.json

@@ -10,8 +10,7 @@
   "scripts": {
     "analyze": "run-s analyze:* format:*",
     "analyze:analyze": "node scripts/workspaces-scripts-bin.mjs analyze",
-    "build": "npm run build:packages && npm run rocket:build",
-    "build:packages": "node scripts/workspaces-scripts-bin.mjs build:package",
+    "build": "npm run rocket:build",
     "build:site": "run-s analyze:* rocket:build",
     "changeset": "changeset",
     "debug": "web-test-runner --watch --config web-test-runner-chrome.config.mjs",
@@ -25,37 +24,30 @@
     "lint:prettier": "node node_modules/prettier/bin-prettier.js \"**/*.{ts,js,mjs,cjs,md}\" --check --ignore-path .eslintignore",
     "lint:types": "npm run types",
     "lint:versions": "node scripts/lint-versions.js",
-    "postinstall": "npm run setup",
-    "release": "changeset publish && yarn format",
+    "postinstall": "npx patch-package",
+    "preview": "node packages/cli/src/cli.js preview --open",
+    "release": "changeset publish && npm run format",
     "rocket:build": "NODE_DEBUG=engine:rendering node --trace-warnings packages/cli/src/cli.js build",
     "rocket:upgrade": "node packages/cli/src/cli.js upgrade",
     "search": "node packages/cli/src/cli.js search",
-    "setup": "npm run setup:ts-configs",
-    "setup:patches": "npx patch-package",
-    "setup:ts-configs": "node scripts/generate-ts-configs.mjs",
-    "start:experimental": "NODE_DEBUG=engine:rendering node --no-warnings --experimental-loader ./packages/engine/src/litCssLoader.js packages/cli/src/cli.js start --open",
     "start": "NODE_DEBUG=engine:rendering node --trace-warnings packages/cli/src/cli.js start --open",
-    "preview": "node packages/cli/src/cli.js preview --open",
-    "test": "yarn test:node && yarn test:web",
+    "start:experimental": "NODE_DEBUG=engine:rendering node --no-warnings --experimental-loader ./packages/engine/src/litCssLoader.js packages/cli/src/cli.js start --open",
+    "test": "npm run test:node && npm run test:web",
     "test:integration": "playwright test packages/*/test-node/*.spec.js --retries=3",
-    "test:node": "yarn test:unit && yarn test:integration",
+    "test:node": "npm run test:unit && npm run test:integration",
     "test:unit": "node --trace-warnings ./node_modules/.bin/mocha --require ./scripts/testMochaGlobalHooks.js \"packages/*/test-node/**/*.test.{ts,js,mjs,cjs}\" -- --timeout 8000 --reporter dot --exit",
     "test:web": "web-test-runner",
-    "types": "run-s types:clear types:copy types:build",
-    "types:build": "tsc --build",
-    "types:clear": "rimraf packages/*/dist-types/",
-    "types:copy": "node scripts/workspaces-scripts-bin.mjs types:copy",
+    "types": "npm run types --workspaces --if-present",
     "update-dependency": "node scripts/update-dependency.js",
-    "update-esm-entrypoints": "node scripts/update-esm-entrypoints.mjs && yarn format",
-    "update-package-configs": "node scripts/update-package-configs.mjs && yarn format",
-    "xprestart": "yarn analyze"
+    "update-esm-entrypoints": "node scripts/update-esm-entrypoints.mjs && npm run format",
+    "update-package-configs": "node scripts/update-package-configs.mjs && npm run format",
+    "xprestart": "npm run analyze"
   },
-  "dependencies": {},
   "devDependencies": {
     "@changesets/cli": "^2.20.0",
     "@custom-elements-manifest/analyzer": "^0.4.12",
-    "@playwright/test": "^1.18.1",
     "@open-wc/testing": "^3.1.2",
+    "@playwright/test": "^1.18.1",
     "@rollup/plugin-commonjs": "^17.0.0",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-typescript": "^8.1.0",
@@ -81,7 +73,7 @@
     "husky": "^4.3.7",
     "lint-staged": "^10.5.3",
     "mocha": "^9.1.3",
-    "node-fetch": "^2.6.7",
+    "node-fetch": "^3.0.0",
     "npm-run-all": "^4.1.5",
     "onchange": "^7.1.0",
     "prettier": "^2.5.1",
@@ -94,7 +86,8 @@
     "rollup-plugin-terser": "^7.0.2",
     "sinon": "^9.2.3",
     "ts-node": "^9.1.1",
-    "typescript": "^4.6.3"
+    "typescript": "^4.8.4",
+    "wireit": "^0.7.2"
   },
   "eslintConfig": {
     "parser": "@typescript-eslint/parser",
@@ -118,7 +111,8 @@
       "@typescript-eslint/no-explicit-any": "off",
       "@typescript-eslint/no-empty-interface": "off",
       "@typescript-eslint/no-unused-vars": "error",
-      "@typescript-eslint/no-var-requires": "off"
+      "@typescript-eslint/no-var-requires": "off",
+      "@typescript-eslint/ban-ts-comment": "off"
     }
   },
   "husky": {

packages/building-rollup/index.js

@@ -1,12 +0,0 @@
-// /**
-//  * @typedef {import('./src/types').BasicOptions} BasicOptions
-//  * @typedef {import('./src/types').SpaOptions} SpaOptions
-//  */
-
-export { createBasicConfig, createBasicMetaConfig } from './src/createBasicConfig.js';
-export { createSpaConfig, createSpaMetaConfig } from './src/createSpaConfig.js';
-export { createMpaConfig, createMpaMetaConfig } from './src/createMpaConfig.js';
-export {
-  createServiceWorkerConfig,
-  createServiceWorkerMetaConfig,
-} from './src/createServiceWorkerConfig.js';

packages/building-rollup/package.json

@@ -13,10 +13,12 @@
   },
   "author": "Modern Web <hello@modern-web.dev> (https://modern-web.dev/)",
   "homepage": "https://rocket.modern-web.dev/docs/tools/building-rollup/",
-  "main": "./index.js",
   "type": "module",
   "exports": {
-    ".": "./index.js"
+    ".": {
+      "types": "./dist-types/src/index.d.ts",
+      "default": "./src/index.js"
+    }
   },
   "scripts": {
     "build:babelrc": "rimraf dist && rollup -c demo/babelrc/rollup.config.js",
@@ -27,6 +29,7 @@
     "build:spa-js-input": "rimraf dist && rollup -c demo/js/rollup.spa-js-input.config.js",
     "build:spa-nomodule": "rimraf dist && rollup -c demo/js/rollup.spa-nomodule.config.js",
     "build:ts": "rimraf dist && rollup -c demo/ts/rollup.spa.config.js",
+    "prepublishOnly": "npm run types",
     "start:babelrc": "npm run build:babelrc && npm run start:build",
     "start:build": "web-dev-server --root-dir dist --compatibility none --open",
     "start:cjs": "npm run build:cjs && npm run start:build",
@@ -38,10 +41,11 @@
     "start:ts": "npm run build:ts && npm run start:build",
     "start:watch": "npm run build:spa-nomodule -- --watch & npm run start:build",
     "test": "npm run test:node",
-    "test:node": "mocha test-node/**/*.test.js --timeout 5000"
+    "test:node": "mocha test-node/**/*.test.js --timeout 5000",
+    "types": "wireit"
   },
   "files": [
-    "*.js",
+    "dist-types",
     "src"
   ],
   "keywords": [
@@ -54,17 +58,35 @@
     "@babel/core": "^7.12.10",
     "@babel/preset-env": "^7.12.11",
     "@rollup/plugin-babel": "^5.2.2",
-    "@rollup/plugin-node-resolve": "^11.0.1",
-    "@rollup/plugin-replace": "^2.4.2",
+    "@rollup/plugin-node-resolve": "^15.0.1",
+    "@rollup/plugin-replace": "^5.0.1",
     "@web/rollup-plugin-html": "^1.8.0",
     "@web/rollup-plugin-import-meta-assets": "^1.0.4",
     "@web/rollup-plugin-polyfills-loader": "^1.1.0",
     "browserslist": "^4.16.1",
+    "plugins-manager": "^0.3.1",
     "rollup-plugin-terser": "^7.0.2",
     "workbox-broadcast-update": "^6.1.5",
     "workbox-cacheable-response": "^6.1.5",
     "workbox-expiration": "^6.1.5",
     "workbox-routing": "^6.1.5",
     "workbox-strategies": "^6.1.5"
+  },
+  "wireit": {
+    "types": {
+      "command": "copyfiles \"./types/**/*.d.ts\" dist-types/ && tsc --build --pretty",
+      "dependencies": [
+        "../plugins-manager:types"
+      ],
+      "clean": "if-file-deleted",
+      "files": [
+        "src/**/*.js",
+        "tsconfig.json"
+      ],
+      "output": [
+        "dist-types/**",
+        ".tsbuildinfo"
+      ]
+    }
   }
 }

packages/building-rollup/src/createBasicConfig.js

@@ -1,4 +1,5 @@
 import resolve from '@rollup/plugin-node-resolve';
+// @ts-ignore
 import { terser } from 'rollup-plugin-terser';
 import babelPkg from '@rollup/plugin-babel';
 
@@ -6,11 +7,19 @@ import { applyPlugins } from 'plugins-manager';
 
 const { babel } = babelPkg;
 
+/** @typedef {import('../types/main.js').BuildingRollupOptions} BuildingRollupOptions */
+
+/**
+ * @param {BuildingRollupOptions} [userConfig]
+ */
 export function createBasicConfig(userConfig) {
   const { config, metaPlugins } = createBasicMetaConfig(userConfig);
   return applyPlugins(config, metaPlugins);
 }
 
+/**
+ * @param {BuildingRollupOptions} [userConfig]
+ */
 export function createBasicMetaConfig(userConfig = { output: {} }) {
   const developmentMode =
     typeof userConfig.developmentMode !== 'undefined'
@@ -37,8 +46,12 @@ export function createBasicMetaConfig(userConfig = { output: {} }) {
     },
   };
 
+  /**
+   * @type {import('plugins-manager').MetaPlugin<any>[]}
+   */
   let metaPlugins = [
     {
+      // @ts-ignore
       plugin: resolve,
       options: {
         moduleDirectories: ['node_modules', 'web_modules'],
@@ -72,6 +85,7 @@ export function createBasicMetaConfig(userConfig = { output: {} }) {
     },
     {
       plugin: terser,
+      options: {},
     },
   ];
 

packages/building-rollup/src/createMpaConfig.js

@@ -1,7 +1,13 @@
 import { createSpaMetaConfig } from './createSpaConfig.js';
 import { adjustPluginOptions, applyPlugins } from 'plugins-manager';
+// @ts-ignore
 import { rollupPluginHTML } from '@web/rollup-plugin-html';
 
+/** @typedef {import('../types/main.js').BuildingRollupOptions} BuildingRollupOptions */
+
+/**
+ * @param {BuildingRollupOptions} [userConfig]
+ */
 export function createMpaConfig(userConfig) {
   const { config, metaPlugins } = createMpaMetaConfig(userConfig);
 
@@ -9,6 +15,9 @@ export function createMpaConfig(userConfig) {
   return final;
 }
 
+/**
+ * @param {BuildingRollupOptions} userConfig
+ */
 export function createMpaMetaConfig(userConfig = { output: {}, setupPlugins: [] }) {
   const { config, metaPlugins } = createSpaMetaConfig(userConfig);
 

packages/building-rollup/src/createServiceWorkerConfig.js

@@ -1,4 +1,5 @@
 import resolve from '@rollup/plugin-node-resolve';
+// @ts-ignore
 import { terser } from 'rollup-plugin-terser';
 import babelPkg from '@rollup/plugin-babel';
 import replace from '@rollup/plugin-replace';
@@ -7,11 +8,19 @@ import { applyPlugins } from 'plugins-manager';
 
 const { babel } = babelPkg;
 
+/** @typedef {import('../types/main.js').BuildingRollupOptions} BuildingRollupOptions */
+
+/**
+ * @param {BuildingRollupOptions} userConfig
+ */
 export function createServiceWorkerConfig(userConfig) {
   const { config, metaPlugins } = createServiceWorkerMetaConfig(userConfig);
   return applyPlugins(config, metaPlugins);
 }
 
+/**
+ * @param {BuildingRollupOptions} userConfig
+ */
 export function createServiceWorkerMetaConfig(userConfig = { output: {} }) {
   const developmentMode =
     typeof userConfig.developmentMode !== 'undefined'
@@ -31,14 +40,19 @@ export function createServiceWorkerMetaConfig(userConfig = { output: {} }) {
     },
   };
 
+  /**
+   * @type {import('plugins-manager').MetaPlugin<any>[]}
+   */
   let metaPlugins = [
     {
+      // @ts-ignore
       plugin: resolve,
       options: {
         moduleDirectories: ['node_modules', 'web_modules'],
       },
     },
     {
+      // @ts-ignore
       plugin: replace,
       options: {
         'process.env.NODE_ENV': JSON.stringify(developmentMode ? 'development' : 'production'),

packages/building-rollup/src/createSpaConfig.js

@@ -1,15 +1,26 @@
+// @ts-ignore
 import { rollupPluginHTML } from '@web/rollup-plugin-html';
+// @ts-ignore
 import { importMetaAssets } from '@web/rollup-plugin-import-meta-assets';
+// @ts-ignore
 import { polyfillsLoader } from '@web/rollup-plugin-polyfills-loader';
 import { applyPlugins } from 'plugins-manager';
 
 import { createBasicMetaConfig } from './createBasicConfig.js';
 
+/** @typedef {import('../types/main.js').BuildingRollupOptions} BuildingRollupOptions */
+
+/**
+ * @param {BuildingRollupOptions} [userConfig]
+ */
 export function createSpaConfig(userConfig) {
   const { config, metaPlugins } = createSpaMetaConfig(userConfig);
   return applyPlugins(config, metaPlugins);
 }
 
+/**
+ * @param {BuildingRollupOptions} userConfig
+ */
 export function createSpaMetaConfig(userConfig = { output: {} }) {
   const { config, metaPlugins, developmentMode } = createBasicMetaConfig(userConfig);
 
@@ -27,8 +38,13 @@ export function createSpaMetaConfig(userConfig = { output: {} }) {
   }
   delete config.absoluteBaseUrl;
 
+  /**
+   * @type {import('plugins-manager').MetaPlugin<any>[]}
+   */
   const spaMetaPlugins = [
+    // @ts-ignore
     ...metaPlugins,
+    // @ts-ignore
     {
       plugin: rollupPluginHTML,
       options: {
@@ -36,9 +52,11 @@ export function createSpaMetaConfig(userConfig = { output: {} }) {
         absoluteBaseUrl,
       },
     },
+    // @ts-ignore
     {
       plugin: importMetaAssets,
     },
+    // @ts-ignore
     {
       plugin: polyfillsLoader,
       options: {

packages/building-rollup/src/index.js

@@ -0,0 +1,11 @@
+/**
+ * @typedef {import('../types/main.js').BuildingRollupOptions} BuildingRollupOptions
+ */
+
+export { createBasicConfig, createBasicMetaConfig } from './createBasicConfig.js';
+export { createSpaConfig, createSpaMetaConfig } from './createSpaConfig.js';
+export { createMpaConfig, createMpaMetaConfig } from './createMpaConfig.js';
+export {
+  createServiceWorkerConfig,
+  createServiceWorkerMetaConfig,
+} from './createServiceWorkerConfig.js';

packages/building-rollup/test-node/createMpaConfig.test.js

@@ -1,6 +1,6 @@
 import chai from 'chai';
 import path from 'path';
-import fs from 'fs';
+import { readFile } from 'fs/promises';
 import { fileURLToPath } from 'url';
 import { rollup } from 'rollup';
 
@@ -8,7 +8,7 @@ const { expect } = chai;
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 /**
- * @param {object} config
+ * @param {import('@rocket/building-rollup').BuildingRollupOptions} config
  */
 async function buildAndWrite(config) {
   const bundle = await rollup(config);
@@ -16,21 +16,27 @@ async function buildAndWrite(config) {
   if (Array.isArray(config.output)) {
     await bundle.write(config.output[0]);
     await bundle.write(config.output[1]);
-  } else {
+  } else if (config.output) {
     await bundle.write(config.output);
   }
 }
 
+/**
+ * @param {string} configString
+ * @returns
+ */
 async function execute(configString) {
   const configPath = path.join(__dirname, 'fixtures', configString.split('/').join(path.sep));
   const config = (await import(configPath)).default;
   await buildAndWrite(config);
 
+  /**
+   * @param {string} fileName
+   */
   return async (fileName, { stripToBody = false, stripStartEndWhitespace = true } = {}) => {
-    let text = await fs.promises.readFile(
-      path.join(config.output.dir, fileName.split('/').join(path.sep)),
-    );
-    text = text.toString();
+    let text = (
+      await readFile(path.join(config.output.dir, fileName.split('/').join(path.sep)))
+    ).toString();
     if (stripToBody) {
       const bodyOpenTagEnd = text.indexOf('>', text.indexOf('<body') + 1) + 1;
       const bodyCloseTagStart = text.indexOf('</body>');

packages/building-rollup/test-node/spa-integration.test-revisit.js

@@ -1,10 +1,11 @@
-/* eslint-disable @typescript-eslint/ban-ts-comment */
 import puppeteer from 'puppeteer';
 import chai from 'chai';
 import path from 'path';
 import fs from 'fs';
+// @ts-ignore
 import rimraf from 'rimraf';
 import { rollup } from 'rollup';
+// @ts-ignore
 import { startDevServer } from '@web/dev-server';
 import { fileURLToPath } from 'url';
 
@@ -14,6 +15,7 @@ const rootDir = path.resolve(__dirname, '..', 'dist');
 const { expect } = chai;
 
 describe('spa integration tests', () => {
+  // @ts-ignore
   let server;
   /** @type {import('puppeteer').Browser} */
   let browser;
@@ -27,6 +29,7 @@ describe('spa integration tests', () => {
       readCliArgs: false,
       readFileConfig: false,
       logStartMessage: false,
+      // @ts-ignore
       clearTerminalOnReload: false,
     });
     browser = await puppeteer.launch();
@@ -35,6 +38,7 @@ describe('spa integration tests', () => {
 
   after(async () => {
     await browser.close();
+    // @ts-ignore
     await server.stop();
   });
 
@@ -45,6 +49,7 @@ describe('spa integration tests', () => {
   ].forEach(testCase => {
     describe(`testcase ${testCase}`, function describe() {
       this.timeout(30000);
+      // @ts-ignore
       let page;
 
       before(async () => {

packages/building-rollup/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../tsconfig.node-base.json",
+  "compilerOptions": {
+    "module": "ESNext",
+    "outDir": "./dist-types",
+    "rootDir": ".",
+    "composite": true,
+    "allowJs": true,
+    "checkJs": true,
+    "emitDeclarationOnly": true,
+    "moduleResolution": "NodeNext"
+  },
+  "include": ["src", "types", "test-node"],
+  "exclude": ["dist-types"]
+}

packages/building-rollup/types/main.d.ts

@@ -0,0 +1,8 @@
+import { RollupOptions } from 'rollup';
+
+interface BuildingRollupOptions extends RollupOptions {
+  developmentMode?: boolean;
+  rootDir?: string;
+  absoluteBaseUrl?: string;
+  setupPlugins?: function[];
+}

packages/check-html-links/CHANGELOG.md

@@ -1,5 +1,21 @@
 # check-html-links
 
+## 0.2.4
+
+### Patch Changes
+
+- 97d5fb2: Add external links validation via the flag `--validate-externals`.
+
+  You can/should provide an optional `--absolute-base-url` to handle urls starting with it as internal.
+
+  ```bash
+  # check external urls
+  npx check-html-links _site --validate-externals
+
+  # check external urls but treat links like https://rocket.modern-web.dev/about/ as internal
+  npx check-html-links _site --validate-externals --absolute-base-url https://rocket.modern-web.dev
+  ```
+
 ## 0.2.3
 
 ### Patch Changes

packages/check-html-links/README.md

@@ -14,7 +14,7 @@ npm i -D check-html-links
 npx check-html-links _site
 ```
 
-For docs please see our homepage [https://rocket.modern-web.dev/docs/tools/check-html-links/](https://rocket.modern-web.dev/docs/tools/check-html-links/).
+For docs please see our homepage [https://rocket.modern-web.dev/tools/check-html-links/overview/](https://rocket.modern-web.dev/tools/check-html-links/overview/).
 
 ## Comparison
 

packages/check-html-links/package.json

@@ -1,6 +1,6 @@
 {
   "name": "check-html-links",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "publishConfig": {
     "access": "public"
   },
@@ -37,6 +37,7 @@
     "command-line-args": "^5.1.1",
     "glob": "^7.0.0",
     "minimatch": "^3.0.4",
+    "node-fetch": "^3.0.0",
     "sax-wasm": "^2.0.0",
     "slash": "^4.0.0"
   },

packages/check-html-links/src/CheckHtmlLinksCli.js

@@ -1,12 +1,10 @@
-/* eslint-disable @typescript-eslint/ban-ts-comment */
-
 /** @typedef {import('../types/main').CheckHtmlLinksCliOptions} CheckHtmlLinksCliOptions */
 
 import path from 'path';
 import chalk from 'chalk';
 
 import commandLineArgs from 'command-line-args';
-import { validateFiles } from './validateFolder.js';
+import { prepareFiles, validateFiles } from './validateFolder.js';
 import { formatErrors } from './formatErrors.js';
 import { listFiles } from './listFiles.js';
 
@@ -18,7 +16,9 @@ export class CheckHtmlLinksCli {
     const mainDefinitions = [
       { name: 'ignore-link-pattern', type: String, multiple: true },
       { name: 'root-dir', type: String, defaultOption: true },
-      { name: 'continue-on-error', type: Boolean, defaultOption: false },
+      { name: 'continue-on-error', type: Boolean },
+      { name: 'validate-externals', type: Boolean },
+      { name: 'absolute-base-url', type: String },
     ];
     const options = commandLineArgs(mainDefinitions, {
       stopAtFirstUnknown: true,
@@ -29,6 +29,8 @@ export class CheckHtmlLinksCli {
       continueOnError: options['continue-on-error'],
       rootDir: options['root-dir'],
       ignoreLinkPatterns: options['ignore-link-pattern'],
+      validateExternals: options['validate-externals'],
+      absoluteBaseUrl: options['absolute-base-url'],
     };
   }
 
@@ -43,22 +45,47 @@ export class CheckHtmlLinksCli {
   }
 
   async run() {
-    const { ignoreLinkPatterns, rootDir: userRootDir } = this.options;
+    const {
+      ignoreLinkPatterns,
+      rootDir: userRootDir,
+      validateExternals,
+      absoluteBaseUrl,
+    } = this.options;
     const rootDir = userRootDir ? path.resolve(userRootDir) : process.cwd();
     const performanceStart = process.hrtime();
 
-    console.log('👀 Checking if all internal links work...');
     const files = await listFiles('**/*.html', rootDir);
 
+    console.log('Check HTML Links');
+
     const filesOutput =
       files.length == 0
-        ? '🧐 No files to check. Did you select the correct folder?'
-        : `🔥 Found a total of ${chalk.green.bold(files.length)} files to check!`;
+        ? '  🧐 No files to check. Did you select the correct folder?'
+        : `  📖 Found ${chalk.green.bold(files.length)} files containing`;
     console.log(filesOutput);
 
-    const { errors, numberLinks } = await validateFiles(files, rootDir, { ignoreLinkPatterns });
+    const { numberLinks, checkLocalFiles, checkExternalLinks } = await prepareFiles(
+      files,
+      rootDir,
+      {
+        ignoreLinkPatterns,
+        validateExternals,
+        absoluteBaseUrl,
+      },
+    );
 
-    console.log(`🔗 Found a total of ${chalk.green.bold(numberLinks)} links to validate!\n`);
+    console.log(`    🔗 ${chalk.green.bold(numberLinks)} internal links`);
+    if (validateExternals) {
+      console.log(`    🌐 ${chalk.green.bold(checkExternalLinks.length)} external links`);
+    }
+
+    console.log('  👀 Checking...');
+
+    const { errors } = await validateFiles({
+      checkLocalFiles,
+      validateExternals,
+      checkExternalLinks,
+    });
 
     const performance = process.hrtime(performanceStart);
     /** @type {string[]} */
@@ -70,7 +97,7 @@ export class CheckHtmlLinksCli {
         referenceCount += error.usage.length;
       }
       output = [
-        `❌ Found ${chalk.red.bold(
+        `  ❌ Found ${chalk.red.bold(
           errors.length.toString(),
         )} missing reference targets (used by ${referenceCount} links) while checking ${
           files.length
@@ -78,7 +105,7 @@ export class CheckHtmlLinksCli {
         ...formatErrors(errors)
           .split('\n')
           .map(line => `  ${line}`),
-        `Checking links duration: ${performance[0]}s ${performance[1] / 1000000}ms`,
+        `  🕑 Checking links duration: ${performance[0]}s ${performance[1] / 1000000}ms`,
       ];
       message = output.join('\n');
       if (this.options.printOnError === true) {
@@ -89,7 +116,7 @@ export class CheckHtmlLinksCli {
       }
     } else {
       console.log(
-        `✅ All internal links are valid. (executed in ${performance[0]}s ${
+        `  ✅ All tested links are valid. (executed in ${performance[0]}s ${
           performance[1] / 1000000
         }ms)`,
       );

packages/check-html-links/src/checkLinks.js

@@ -0,0 +1,63 @@
+import fetch from 'node-fetch';
+
+/**
+ * @type {Map<string,boolean>}
+ */
+const resultsMap = new Map();
+
+/**
+ *
+ * @param {string} url
+ * @param {boolean} result
+ * @returns {boolean}
+ */
+const memorizeCheckup = (url, result) => {
+  resultsMap.set(url, result);
+  return result;
+};
+
+/**
+ *
+ * @param {string} url
+ * @param {string} method
+ * @returns
+ */
+const fetchWrap = async (url, method = 'GET') => {
+  return Promise.race([
+    fetch(url, { method })
+      .then(response => response.ok)
+      .catch(() => false),
+    new Promise(resolve => setTimeout(resolve, 10000, false)),
+  ]);
+};
+
+/**
+ *
+ * @param {string} url
+ * @returns {Promise<boolean>}
+ */
+const fetchHead = async url => fetchWrap(url, 'HEAD');
+
+/**
+ *
+ * @param {string} url - URL object to check
+ * @returns {Promise<boolean>} true if url is alive or false if not
+ */
+const checkUrl = async url =>
+  (fetchHead(url) || fetchWrap(url)).then(result => memorizeCheckup(url, result));
+
+/**
+ *
+ * @param {string} link - link string to check
+ * @returns {Promise<boolean>}
+ */
+export const checkLink = async link => {
+  const url = link.startsWith('//') ? `https:${link}` : link;
+  return resultsMap.get(url) ?? checkUrl(url);
+};
+/**
+ * Check an array of links and return an object with
+ *
+ * @param {string[]} links Links to check
+ */
+export const checkLinks = async links => Promise.all(links.map(checkLink));

packages/check-html-links/src/formatErrors.js

@@ -15,7 +15,7 @@ export function formatErrors(errors, relativeFrom = process.cwd()) {
     const filePath = path.relative(relativeFrom, error.filePath);
     if (error.onlyAnchorMissing === true) {
       output.push(
-        `${number}. missing ${chalk.red.bold(
+        `  ${number}. missing ${chalk.red.bold(
           `id="${error.usage[0].anchor}"`,
         )} in ${chalk.cyanBright(filePath)}`,
       );
@@ -24,7 +24,7 @@ export function formatErrors(errors, relativeFrom = process.cwd()) {
       const title =
         firstAttribute === 'src' || firstAttribute === 'srcset' ? 'file' : 'reference target';
 
-      output.push(`${number}. missing ${title} ${chalk.red.bold(filePath)}`);
+      output.push(`  ${number}. missing ${title} ${chalk.red.bold(filePath)}`);
     }
     const usageLength = error.usage.length;
 
@@ -34,11 +34,11 @@ export function formatErrors(errors, relativeFrom = process.cwd()) {
       const clickAbleLink = chalk.cyanBright(`${usagePath}:${usage.line + 1}:${usage.character}`);
       const attributeStart = chalk.gray(`${usage.attribute}="`);
       const attributeEnd = chalk.gray('"');
-      output.push(`  from ${clickAbleLink} via ${attributeStart}${usage.value}${attributeEnd}`);
+      output.push(`    from ${clickAbleLink} via ${attributeStart}${usage.value}${attributeEnd}`);
     }
     if (usageLength > 3) {
       const more = chalk.red((usageLength - 3).toString());
-      output.push(`  ... ${more} more references to this target`);
+      output.push(`    ... ${more} more references to this target`);
     }
     output.push('');
   }

packages/check-html-links/src/validateFolder.js

@@ -1,12 +1,11 @@
-/* eslint-disable @typescript-eslint/ban-ts-comment */
 import fs from 'fs';
 import saxWasm from 'sax-wasm';
 import minimatch from 'minimatch';
 import { createRequire } from 'module';
-
-import { listFiles } from './listFiles.js';
 import path from 'path';
 import slash from 'slash';
+import { listFiles } from './listFiles.js';
+import { checkLinks } from './checkLinks.js';
 
 /** @typedef {import('../types/main').Link} Link */
 /** @typedef {import('../types/main').LocalFile} LocalFile */
@@ -28,6 +27,9 @@ const parserIds = new SAXParser(SaxEventType.Attribute, streamOptions);
 /** @type {Error[]} */
 let checkLocalFiles = [];
 
+/** @type {Error[]} */
+let checkExternalLinks = [];
+
 /** @type {Error[]} */
 let errors = [];
 
@@ -151,6 +153,26 @@ function addLocalFile(filePath, anchor, usageObj) {
   }
 }
 
+/**
+ * @param {string} filePath
+ * @param {Usage} usageObj
+ */
+function addExternalLink(filePath, usageObj) {
+  const foundIndex = checkExternalLinks.findIndex(item => {
+    return item.filePath === filePath;
+  });
+
+  if (foundIndex === -1) {
+    checkExternalLinks.push({
+      filePath,
+      onlyAnchorMissing: false,
+      usage: [usageObj],
+    });
+  } else {
+    checkExternalLinks[foundIndex].usage.push(usageObj);
+  }
+}
+
 /**
  * @param {string} inValue
  */
@@ -200,11 +222,16 @@ function isNonHttpSchema(url) {
  * @param {object} options
  * @param {string} options.htmlFilePath
  * @param {string} options.rootDir
+ * @param {string} options.absoluteBaseUrl
  * @param {function(string): boolean} options.ignoreUsage
  */
-async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage }) {
+async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage, absoluteBaseUrl }) {
   for (const hrefObj of links) {
-    const { value, anchor } = getValueAndAnchor(hrefObj.value);
+    const { value: rawValue, anchor } = getValueAndAnchor(hrefObj.value);
+
+    const value = rawValue.startsWith(absoluteBaseUrl)
+      ? rawValue.substring(absoluteBaseUrl.length)
+      : rawValue;
 
     const usageObj = {
       attribute: hrefObj.attribute,
@@ -229,8 +256,7 @@ async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage }) {
     } else if (valueFile === '' && anchor !== '') {
       addLocalFile(htmlFilePath, anchor, usageObj);
     } else if (value.startsWith('//') || value.startsWith('http')) {
-      // TODO: handle external urls
-      // external url - we do not handle that (yet)
+      addExternalLink(htmlFilePath, usageObj);
     } else if (value.startsWith('/')) {
       const filePath = path.join(rootDir, valueFile);
       addLocalFile(filePath, anchor, usageObj);
@@ -244,7 +270,7 @@ async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage }) {
     }
   }
 
-  return { checkLocalFiles: [...checkLocalFiles] };
+  return { checkLocalFiles: [...checkLocalFiles], checkExternalLinks: [...checkExternalLinks] };
 }
 
 /**
@@ -283,17 +309,34 @@ async function validateLocalFiles(checkLocalFiles) {
   return errors;
 }
 
+/**
+ *
+ * @param {Error[]} checkExternalLinks
+ */
+async function validateExternalLinks(checkExternalLinks) {
+  for await (const localFileObj of checkExternalLinks) {
+    const links = localFileObj.usage.map(usage => usage.value);
+    const results = await checkLinks(links);
+    localFileObj.usage = localFileObj.usage.filter((link, index) => !results[index]);
+    if (localFileObj.usage.length > 0) {
+      errors.push(localFileObj);
+    }
+  }
+  return errors;
+}
+
 /**
  * @param {string[]} files
  * @param {string} rootDir
  * @param {Options} opts?
  */
-export async function validateFiles(files, rootDir, opts) {
+export async function prepareFiles(files, rootDir, opts) {
   await parserReferences.prepareWasm(saxWasmBuffer);
   await parserIds.prepareWasm(saxWasmBuffer);
 
   errors = [];
   checkLocalFiles = [];
+  checkExternalLinks = [];
   idCache = new Map();
   let numberLinks = 0;
 
@@ -309,11 +352,27 @@ export async function validateFiles(files, rootDir, opts) {
   for (const htmlFilePath of files) {
     const { links } = await extractReferences(htmlFilePath);
     numberLinks += links.length;
-    await resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage });
+    await resolveLinks(links, {
+      htmlFilePath,
+      rootDir,
+      ignoreUsage,
+      absoluteBaseUrl: opts?.absoluteBaseUrl,
+    });
   }
-  await validateLocalFiles(checkLocalFiles);
+  return { checkLocalFiles, checkExternalLinks, numberLinks };
+}
 
-  return { errors: errors, numberLinks: numberLinks };
+/**
+ * @param {*} param0
+ * @returns
+ */
+export async function validateFiles({ checkLocalFiles, validateExternals, checkExternalLinks }) {
+  await validateLocalFiles(checkLocalFiles);
+  if (validateExternals) {
+    await validateExternalLinks(checkExternalLinks);
+  }
+
+  return { errors };
 }
 
 /**
@@ -323,6 +382,14 @@ export async function validateFiles(files, rootDir, opts) {
 export async function validateFolder(inRootDir, opts) {
   const rootDir = path.resolve(inRootDir);
   const files = await listFiles('**/*.html', rootDir);
-  const { errors } = await validateFiles(files, rootDir, opts);
+
+  const { checkLocalFiles, checkExternalLinks } = await prepareFiles(files, rootDir, opts);
+
+  const { errors } = await validateFiles({
+    checkLocalFiles,
+    validateExternals: opts?.validateExternals,
+    checkExternalLinks,
+  });
+
   return errors;
 }

packages/check-html-links/test-node/fixtures/external-link/index.html

@@ -1,5 +1,9 @@
-<!-- ignore known subsystems -->
-<a href="/docs/"></a>
-<a href="/developer/getting-started.html#js"></a>
-<a href="/developer/language-guides/"></a>
-<a href="/developer/javascript.html"></a>
+<!-- valid -->
+<a href="//rocket.modern-web.dev/"></a>
+<a href="http://rocket.modern-web.dev/"></a>
+<a href="https://rocket.modern-web.dev/"></a>
+
+<!-- invalid -->
+<a href="//rocket.modern-web.dev/unexists-page/"></a>
+<a href="http://rocket.modern-web.dev/unexists-page/"></a>
+<a href="https://rocket.modern-web.dev/unexists-page/"></a>

packages/check-html-links/test-node/fixtures/internal-link/index.html

@@ -5,8 +5,5 @@
 <a href="./page.html"></a>
 <a href=" ./page.html "></a>
 <a href=" /page.html "></a>
-<a href="//domain.com/something/"></a>
-<a href="http://domain.com/something/"></a>
-<a href="https://domain.com/something/"></a>
 <a href=""></a>
 <a href=":~:text=put%20your%20labels%20above%20your%20inputs">Sign-in form best practices</a>

packages/check-html-links/test-node/fixtures/internal-own-absolute-base-path/about.html

@@ -0,0 +1 @@
+About

packages/check-html-links/test-node/fixtures/internal-own-absolute-base-path/index.html

@@ -0,0 +1,2 @@
+<a href="about.html">About</a>
+<a href="http://localhost/about.html">About Absolute</a>

packages/check-html-links/test-node/formatErrors.test.js

@@ -20,19 +20,19 @@ describe('formatErrors', () => {
     const result = await executeAndFormat('fixtures/test-case');
     expect(result.trim().split('\n')).to.deep.equal([
       '1. missing id="my-teams" in fixtures/test-case/price/index.html',
-      '  from fixtures/test-case/history/index.html:1:9 via href="/price/#my-teams"',
+      '    from fixtures/test-case/history/index.html:1:9 via href="/price/#my-teams"',
       '',
-      '2. missing file fixtures/test-case/about/images/team.png',
-      '  from fixtures/test-case/about/index.html:3:10 via src="./images/team.png"',
+      '  2. missing file fixtures/test-case/about/images/team.png',
+      '    from fixtures/test-case/about/index.html:3:10 via src="./images/team.png"',
       '',
-      '3. missing reference target fixtures/test-case/aboot',
-      '  from fixtures/test-case/about/index.html:6:11 via href="/aboot"',
-      '  from fixtures/test-case/history/index.html:4:11 via href="/aboot"',
-      '  from fixtures/test-case/index.html:4:11 via href="/aboot"',
-      '  ... 2 more references to this target',
+      '  3. missing reference target fixtures/test-case/aboot',
+      '    from fixtures/test-case/about/index.html:6:11 via href="/aboot"',
+      '    from fixtures/test-case/history/index.html:4:11 via href="/aboot"',
+      '    from fixtures/test-case/index.html:4:11 via href="/aboot"',
+      '    ... 2 more references to this target',
       '',
-      '4. missing reference target fixtures/test-case/prce',
-      '  from fixtures/test-case/index.html:1:9 via href="./prce"',
+      '  4. missing reference target fixtures/test-case/prce',
+      '    from fixtures/test-case/index.html:1:9 via href="./prce"',
     ]);
   });
 });

packages/check-html-links/test-node/validateFolder.test.js

@@ -66,6 +66,74 @@ describe('validateFolder', () => {
     ]);
   });
 
+  it('validates external links', async () => {
+    const { errors, cleanup } = await execute('fixtures/external-link', {
+      validateExternals: true,
+    });
+    expect(cleanup(errors)).to.deep.equal([
+      {
+        filePath: 'fixtures/external-link/index.html',
+        onlyAnchorMissing: false,
+        usage: [
+          {
+            attribute: 'href',
+            value: '//rocket.modern-web.dev/unexists-page/',
+            file: 'fixtures/external-link/index.html',
+            line: 6,
+            character: 9,
+            anchor: '',
+          },
+          {
+            attribute: 'href',
+            value: 'http://rocket.modern-web.dev/unexists-page/',
+            file: 'fixtures/external-link/index.html',
+            line: 7,
+            character: 9,
+            anchor: '',
+          },
+          {
+            attribute: 'href',
+            value: 'https://rocket.modern-web.dev/unexists-page/',
+            file: 'fixtures/external-link/index.html',
+            line: 8,
+            character: 9,
+            anchor: '',
+          },
+        ],
+      },
+    ]);
+  });
+
+  it('validates links with own absolute base url as internal', async () => {
+    const { errors, cleanup } = await execute('fixtures/internal-own-absolute-base-path', {
+      validateExternals: true,
+      absoluteBaseUrl: 'http://localhost',
+    });
+    expect(cleanup(errors)).to.deep.equal([]);
+  });
+
+  it('validates all full urls if there is no absoluteBaseUrl provided', async () => {
+    const { errors, cleanup } = await execute('fixtures/internal-own-absolute-base-path', {
+      validateExternals: true,
+    });
+    expect(cleanup(errors)).to.deep.equal([
+      {
+        filePath: 'fixtures/internal-own-absolute-base-path/index.html',
+        onlyAnchorMissing: false,
+        usage: [
+          {
+            anchor: '',
+            attribute: 'href',
+            character: 9,
+            file: 'fixtures/internal-own-absolute-base-path/index.html',
+            line: 1,
+            value: 'http://localhost/about.html',
+          },
+        ],
+      },
+    ]);
+  });
+
   it('groups multiple usage of the same missing file', async () => {
     const { errors, cleanup } = await execute('fixtures/internal-links-to-same-file');
     expect(cleanup(errors)).to.deep.equal([

packages/check-html-links/types/main.d.ts

@@ -26,13 +26,15 @@ export interface Error {
   usage: Usage[];
 }
 
-interface Options {
+export interface Options {
   ignoreLinkPatterns: string[] | null;
+  validateExternals: boolean;
+  absoluteBaseUrl: string;
 }
 
-export interface CheckHtmlLinksCliOptions {
+export interface CheckHtmlLinksCliOptions extends Options {
   printOnError: boolean;
   rootDir: string;
-  ignoreLinkPatterns: string[] | null;
   continueOnError: boolean;
+  absoluteBaseUrl: string;
 }

packages/check-website/CHANGELOG.md

@@ -0,0 +1,87 @@
+# check-html-links
+
+## 0.2.4
+
+### Patch Changes
+
+- 97d5fb2: Add external links validation via the flag `--validate-externals`.
+
+  You can/should provide an optional `--absolute-base-url` to handle urls starting with it as internal.
+
+  ```bash
+  # check external urls
+  npx check-html-links _site --validate-externals
+
+  # check external urls but treat links like https://rocket.modern-web.dev/about/ as internal
+  npx check-html-links _site --validate-externals --absolute-base-url https://rocket.modern-web.dev
+  ```
+
+## 0.2.3
+
+### Patch Changes
+
+- 5043429: Ignore `<a href="tel:9999">` links
+- f08f926: Add missing `slash` dependency
+- a0e8edf: Ignore links containing not http schema urls like `sketch://`, `vscode://`, ...
+
+  ```html
+  <a href="sketch://add-library?url=https%3A%2F%2Fmyexample.com%2Fdesign%2Fui-kit.xml"></a>
+  <a href="vscode://file/c:/myProject/package.json:5:10"></a>
+  ```
+
+- 1949b1e: Ignore plain and html encoded mailto links
+
+  ```html
+  <!-- source -->
+  <a href="mailto:address@example.com">contact</a>
+
+  <!-- html encoded -->
+  <a
+    href="&#109;&#97;&#105;&#108;&#116;&#111;&#58;&#97;&#100;&#100;&#114;&#101;&#115;&#115;&#64;&#101;&#120;&#97;&#109;&#112;&#108;&#101;&#46;&#99;&#111;&#109;"
+    >contact</a
+  >
+  ```
+
+## 0.2.2
+
+### Patch Changes
+
+- 66c2d78: fix: windows path issue
+
+## 0.2.1
+
+### Patch Changes
+
+- be0d0b3: fix: add missing main entry to the packages
+
+## 0.2.0
+
+### Minor Changes
+
+- 750418b: Uses a class for the CLI and adding the following options:
+
+  - `--root-dir` the root directory to serve files from. Defaults to the current working directory
+  - `--ignore-link-pattern` do not check links matching the pattern
+  - `--continue-on-error` if present it will not exit with an error code - useful while writing or for temporary passing a ci
+
+  BREAKING CHANGE:
+
+  - Exists with an error code if a broken link is found
+
+## 0.1.2
+
+### Patch Changes
+
+- f343c50: When reading bigger files, especially bigger files with all content on one line it could mean a read chunk is in the middle of a character. This can lead to strange symbols in the resulting string. The `hightWaterMark` is now increased from the node default of 16KB to 256KB. Additionally, the `hightWaterMark` is now synced for reading and parsing.
+
+## 0.1.1
+
+### Patch Changes
+
+- eb74110: Add info about how many files and links will be checked
+
+## 0.1.0
+
+### Minor Changes
+
+- cd22231: Initial release

packages/check-website/README.md

@@ -0,0 +1,28 @@
+# Check Website
+
+A fast checker for broken links/references in HTML.
+
+## Installation
+
+```shell
+npm i -D check-website
+```
+
+## Usage
+
+```bash
+npx check-website
+```
+
+For docs please see our homepage [https://rocket.modern-web.dev/tools/check-html-links/overview/](https://rocket.modern-web.dev/tools/check-html-links/overview/).
+
+## Comparison
+
+Checking the output of the [11ty-website](https://github.com/11ty/11ty-website) with 13 missing reference targets (used by 516 links) while checking 501 files. (on January 17, 2021)
+
+| Tool             | Lines printed | Times  | Lang | Dependency Tree |
+| ---------------- | ------------- | ------ | ---- | --------------- |
+| check-html-links | 38            | ~2.5s  | node | 19              |
+| link-checker     | 3000+         | ~11s   | node | 106             |
+| hyperlink        | 68            | 4m 20s | node | 481             |
+| htmltest         | 1000+         | ~0.7s  | GO   | -               |

packages/check-website/TODO.md

@@ -0,0 +1,3 @@
+# Check HTML Links
+
+- Support external entrypoints... e.g. user tried this `npx check-html-links@latest https://jasik.xyz`

packages/check-website/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "check-website",
+  "version": "0.0.1",
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "A fast low dependency checker of html links/references",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/modernweb-dev/rocket.git",
+    "directory": "packages/check-website"
+  },
+  "author": "Modern Web <hello@modern-web.dev> (https://modern-web.dev/)",
+  "homepage": "https://rocket.modern-web.dev/docs/tools/check-website/",
+  "bin": {
+    "check-website": "src/cli.js"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist-types/index.d.ts",
+      "default": "./src/index.js"
+    }
+  },
+  "scripts": {
+    "test": "mocha --timeout 5000 test-node/**/*.test.js test-node/*.test.js",
+    "test:watch": "onchange 'src/**/*.js' 'test-node/**/*.js' -- npm test",
+    "types": "wireit"
+  },
+  "files": [
+    "dist-types",
+    "src"
+  ],
+  "dependencies": {
+    "cacheable-lookup": "^7.0.0",
+    "colorette": "^2.0.16",
+    "commander": "^9.0.0",
+    "got": "^12.5.2",
+    "mime-types": "^2.1.35",
+    "minimatch": "^3.0.4",
+    "node-fetch": "^3.0.0",
+    "normalize-url": "^7.2.0",
+    "p-queue": "^7.3.0",
+    "sax-wasm": "^2.0.0",
+    "slash": "^4.0.0"
+  },
+  "devDependencies": {
+    "@types/mime-types": "^2.1.1"
+  },
+  "wireit": {
+    "types": {
+      "command": "tsc --build --pretty",
+      "dependencies": [
+        "../plugins-manager:types"
+      ],
+      "files": [
+        "src/**/*.js",
+        "test-node/**/*.js",
+        "types",
+        "tsconfig.json"
+      ],
+      "output": [
+        "dist-types/**"
+      ]
+    }
+  }
+}

packages/check-website/src/CheckWebsiteCli.js

@@ -0,0 +1,217 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment */
+
+/** @typedef {import('../types/main.js').FullCheckWebsiteCliOptions} FullCheckWebsiteCliOptions */
+/** @typedef {import('../types/main.js').CheckWebsiteCliOptions} CheckWebsiteCliOptions */
+/** @typedef {import('./assets/Asset.js').Asset} Asset */
+
+import { EventEmitter } from 'events';
+import path from 'path';
+import { Command } from 'commander';
+import { green } from 'colorette';
+import { gatherFiles } from './helpers/gatherFiles.js';
+import { renderProgressBar } from './cli/renderProgressBar.js';
+import { LocalReferencesPlugin } from './plugins/LocalReferencesPlugin.js';
+// import { HasCanonicalPlugin } from './plugins/HasCanonicalPlugin.js';
+import { ExternalReferencesPlugin } from './plugins/ExternalReferencesPlugin.js';
+import { AssetManager } from './assets/AssetManager.js';
+import { LitTerminal } from './cli/LitTerminal.js';
+import { cli } from './cli/cli.js';
+import { IssueManager } from './issues/IssueManager.js';
+import { hr } from './cli/helpers.js';
+import { existsSync } from 'fs';
+import { readFile } from 'fs/promises';
+import { pathToFileURL } from 'url';
+import { normalizeUrl } from './helpers/normalizeUrl.js';
+import { HtmlPage } from './assets/HtmlPage.js';
+
+export class CheckWebsiteCli extends LitTerminal {
+  /** @type {FullCheckWebsiteCliOptions} */
+  options = {
+    configFile: '',
+    inputDir: process.cwd(),
+    originUrl: '',
+    assetManager: new AssetManager(),
+    issueManager: new IssueManager(),
+    plugins: [],
+  };
+
+  /**
+   * @readonly
+   */
+  events = new EventEmitter();
+
+  constructor({ argv = process.argv } = {}) {
+    super();
+    this.argv = argv;
+
+    this.program = new Command();
+    this.program.allowUnknownOption().option('-c, --config-file <path>', 'path to config file');
+    this.program.parse(this.argv);
+
+    this.program.allowUnknownOption(false);
+
+    if (this.program.opts().configFile) {
+      this.options.configFile = this.program.opts().configFile;
+    }
+
+    this.program
+      .option('-i, --input-dir <path>', 'path to where to search for source files')
+      .action(async cliOptions => {
+        this.setOptions(cliOptions);
+      });
+
+    this.options.plugins = [
+      //
+      new LocalReferencesPlugin(),
+      // new HasCanonicalPlugin(),
+      new ExternalReferencesPlugin(),
+    ];
+
+    /** @param {string} msg */
+    this.options.issueManager.logger = msg => this.logStatic(msg);
+  }
+
+  /**
+   * @param {Partial<CheckWebsiteCliOptions>} newOptions
+   */
+  setOptions(newOptions) {
+    this.options = {
+      ...this.options,
+      ...newOptions,
+    };
+  }
+
+  async applyConfigFile() {
+    if (this.options.configFile) {
+      const configFilePath = path.resolve(this.options.configFile);
+      const fileOptions = (await import(configFilePath)).default;
+      this.setOptions(fileOptions);
+    } else {
+      // make sure all default settings are properly initialized
+      this.setOptions({});
+    }
+  }
+
+  async execute() {
+    super.execute();
+    await this.applyConfigFile();
+    // const inputDir = userInputDir ? path.resolve(userInputDir) : process.cwd();
+
+    let entrypoint = path.join(this.options.inputDir, 'index.html');
+    if (!existsSync(entrypoint)) {
+      console.log(`Entrypoint ${entrypoint} does not exist`);
+      process.exit(1);
+    }
+    if (!this.options.originUrl) {
+      const entryHtml = await readFile(entrypoint, 'utf-8');
+      const canonicalUrl = findCanonicalUrl(entryHtml);
+      if (canonicalUrl) {
+        this.options.originUrl = canonicalUrl;
+      } else {
+        console.log(`No canonical url found in ${entrypoint}`);
+        process.exit(1);
+      }
+    }
+
+    if (!this.options.isLocalUrl) {
+      /** @param {string} url */
+      this.options.isLocalUrl = url => {
+        const normalizedUrl = normalizeUrl(url);
+        return normalizedUrl.startsWith(this.options.originUrl);
+      };
+    }
+
+    this.logStatic('🔎 Check Website');
+    this.logStatic('');
+    this.logStatic(`👀 Start crawling from ${green('index.html')}`);
+    this.logStatic(`📄 Will follow all links within ${green(this.options.originUrl)}`);
+    this.logStatic('');
+
+    const onParseElementCallbacks = this.options.plugins
+      .map(plugin => plugin.onParseElement)
+      .filter(Boolean);
+
+    this.options.assetManager = new AssetManager({
+      originPath: this.options.inputDir,
+      originUrl: this.options.originUrl,
+      // TODO: fix type...
+      // @ts-ignore
+      onParseElementCallbacks,
+      plugins: this.options.plugins,
+      isLocalUrl: this.options.isLocalUrl,
+      skips: this.options.skips,
+    });
+    const pluginsAndAssetManager = [...this.options.plugins, this.options.assetManager];
+
+    this.options.assetManager.events.on('idle', () => {
+      if (pluginsAndAssetManager.every(p => p.isIdle)) {
+        this.updateComplete.then(() => {
+          this.events.emit('done');
+        });
+      }
+    });
+
+    for (const plugin of this.options.plugins) {
+      plugin.assetManager = this.options.assetManager;
+      plugin.issueManager = this.options.issueManager;
+      plugin.isLocalUrl = this.options.isLocalUrl;
+      await plugin.setup(this);
+      plugin.events.on('progress', () => {
+        this.requestUpdate();
+      });
+      plugin.events.on('idle', () => {
+        if (pluginsAndAssetManager.every(p => p.isIdle)) {
+          this.updateComplete.then(() => {
+            this.events.emit('done');
+          });
+        }
+      });
+    }
+
+    const files = await gatherFiles(this.options.inputDir);
+    if (files.length === 0) {
+      console.log('🧐 No files to check. Did you select the correct folder?');
+      process.exit(1);
+    }
+
+    for (const [, file] of files.entries()) {
+      const fileUrl = pathToFileURL(file);
+      this.options.assetManager.addExistingFile(fileUrl);
+    }
+
+    // start crawling at the main index.html
+    const rootPage = this.options.assetManager.get(this.options.originUrl + '/index.html');
+    if (rootPage && rootPage instanceof HtmlPage) {
+      await rootPage.parse();
+    }
+  }
+
+  renderParsing() {
+    const title = `Parsing:`.padEnd(11);
+    const total = this.options.assetManager.parsingQueue.getTotal();
+    const doneNr = this.options.assetManager.parsingQueue.getDone();
+    const duration = this.options.assetManager.parsingQueue.getDuration();
+    const progress = renderProgressBar(doneNr, 0, total);
+    const minNumberLength = `${total}`.length;
+    const done = `${doneNr}`.padStart(minNumberLength);
+    return `${title} ${progress} ${done}/${total} files | 🕑 ${duration}s`;
+  }
+
+  render() {
+    return cli`
+      \n${hr()}
+      ${this.renderParsing()}
+      ${this.options.plugins.map(plugin => plugin.render()).join('\n')}
+    `;
+  }
+}
+
+/**
+ * @param {string} html
+ */
+function findCanonicalUrl(html) {
+  const matches = html.match(/<link\s*rel="canonical"\s*href="(.*)"/);
+  if (matches) {
+    return normalizeUrl(matches[1]);
+  }
+}

packages/check-website/src/assets/Asset.js

@@ -0,0 +1,138 @@
+import { EventEmitter } from 'events';
+import fetch from 'node-fetch';
+import got, { RequestError } from 'got';
+import CacheableLookup from 'cacheable-lookup';
+
+const dnsCache = new CacheableLookup();
+
+/** @typedef {import('../../types/main.js').AssetStatus} AssetStatus */
+
+export const ASSET_STATUS = /** @type {const} */ ({
+  unknown: 0,
+
+  // 200+ exists
+  exists: 200,
+  existsLocal: 220,
+  existsExternal: 230,
+
+  // 250+ parsed
+  parsed: 250,
+  parsedLocal: 260,
+  parsedExternal: 270,
+
+  // 300+ redirect
+  redirect: 300,
+
+  // 400+ error
+  missing: 400,
+});
+
+export class Asset {
+  /**
+   * The full page URL.
+   *
+   * Example: `https://docs.astro.build/en/getting-started/`
+   * @type {URL}
+   */
+  url;
+
+  /**
+   * @type {AssetStatus}
+   */
+  #status = ASSET_STATUS.unknown;
+
+  /**
+   * @param {AssetStatus} status
+   */
+  set status(status) {
+    this.#status = status;
+    this.events.emit('status-changed');
+  }
+
+  get status() {
+    return this.#status;
+  }
+
+  /**
+   * @readonly
+   */
+  events = new EventEmitter();
+
+  localPath = '';
+
+  localSourcePath = '';
+
+  /** @type {import('../../types/main.js').FullAssetOptions} */
+  options = {
+    originUrl: '',
+    originPath: '',
+    localPath: '',
+    localSourcePath: '',
+    fetch,
+    assetManager: undefined,
+    isLocalUrl: url => url.startsWith(this.options.originUrl),
+    skip: false,
+  };
+
+  /**
+   *
+   * @param {URL} url
+   * @param {import('../../types/main.js').AssetOptions} options
+   */
+  constructor(url, options) {
+    this.options = { ...this.options, ...options };
+    this.url = url;
+    this.localPath = this.options.localPath;
+
+    if (this.url.protocol === 'file:') {
+      throw new Error(`File protocol is not supported. Used by ${this.url.href}`);
+    }
+  }
+
+  exists() {
+    return new Promise(resolve => {
+      if (this.status > ASSET_STATUS.unknown && this.status < ASSET_STATUS.missing) {
+        resolve(true);
+      } else if (this.options.isLocalUrl(this.url.href)) {
+        // Local assets need to be added upfront and are not dynamically discovered - potentially a feature for later
+        resolve(false);
+      } else {
+        this.options.assetManager?.fetchQueue
+          .add(async () => await this.executeExists())
+          .finally(() => {
+            resolve(this.status > ASSET_STATUS.unknown && this.status < ASSET_STATUS.missing);
+          });
+      }
+    });
+  }
+
+  async executeExists() {
+    if (this.options.skip) {
+      return;
+    }
+    // TODO: detect server redirects (301, 302, etc)?
+    // const fetching = fetch(this.url.href, { method: 'HEAD', redirect: "error" });
+    try {
+      await got(this.url.href, {
+        method: 'HEAD',
+        retry: {
+          limit: 1,
+          calculateDelay: ({ computedValue }) => {
+            return computedValue / 100;
+          },
+        },
+        dnsCache,
+      });
+      this.status = ASSET_STATUS.existsExternal;
+    } catch (err) {
+      if (err instanceof RequestError) {
+        this.status =
+          /** @type {AssetStatus} */ (err?.response?.statusCode) || ASSET_STATUS.missing;
+      }
+    }
+  }
+
+  isLocal() {
+    return this.status === ASSET_STATUS.existsLocal || this.status === ASSET_STATUS.parsedLocal;
+  }
+}

packages/check-website/src/assets/AssetManager.js

@@ -0,0 +1,220 @@
+import fetch from 'node-fetch';
+import path from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
+import mime from 'mime-types';
+
+import { Asset, ASSET_STATUS } from './Asset.js';
+import { HtmlPage } from './HtmlPage.js';
+import { normalizeUrl, normalizeToLocalUrl } from '../helpers/normalizeUrl.js';
+import { decodeNumberHtmlEntities } from '../helpers/decodeNumberHtmlEntities.js';
+import { Queue } from '../helpers/Queue.js';
+import EventEmitter from 'events';
+import minimatch from 'minimatch';
+
+/** @typedef {import('../plugins/Plugin.js').Plugin} Plugin */
+
+const classMap = {
+  Asset,
+  HtmlPage,
+};
+
+/**
+ * @param {string} url
+ * @returns {URL}
+ */
+function getUrl(url) {
+  try {
+    return new URL(url);
+  } catch (e) {
+    // handle html encoded mailto links like <a href="&#109;&#97;&#105;&#108;&#116;&#111;&#58;">
+    const decoded = decodeNumberHtmlEntities(url);
+    return new URL(decoded);
+  }
+}
+
+export class AssetManager {
+  /** @type {Map<string, Asset | HtmlPage>} */
+  assets = new Map();
+
+  /**
+   * @readonly
+   */
+  events = new EventEmitter();
+
+  /** Queue *************************/
+  parsingQueue = new Queue({
+    concurrency: 1,
+  });
+
+  fetchQueue = new Queue({
+    concurrency: 1,
+    carryoverConcurrencyCount: true,
+    interval: 500,
+    intervalCap: 10,
+  });
+
+  /** @type {import('../../types/main.js').FullAssetManagerOptions} */
+  options = {
+    originUrl: '',
+    originPath: '',
+    fetch,
+    plugins: [],
+    isLocalUrl: url => url.startsWith(this.options.originUrl),
+    onParseElementCallbacks: [],
+  };
+
+  /**
+   * @param {import('../../types/main.js').AssetManagerOptions} [options]
+   */
+  constructor(options) {
+    this.options = { ...this.options, ...options };
+
+    this.parsingQueue.on('idle', () => {
+      this.events.emit('idle');
+    });
+    this.fetchQueue.on('idle', () => {
+      this.events.emit('idle');
+    });
+  }
+
+  /**
+   * @param {string} localPath
+   * @returns {Asset | HtmlPage}
+   */
+  addFile(localPath) {
+    const url = pathToFileURL(localPath); // new URL('file://' + localPath);
+    return this.addUrl(url);
+  }
+
+  /**
+   * It does not check if the asset actually exits.
+   * ONLY call it for asset urls you know exist.
+   *
+   * @param {URL} fileUrl
+   * @return {Asset | HtmlPage}
+   */
+  addExistingFile(fileUrl) {
+    const filePath = fileURLToPath(fileUrl);
+    const rel = path.relative(this.options.originPath, filePath);
+    const url = new URL(rel, this.options.originUrl);
+    const localPath = path.join(this.options.originPath, rel);
+    if (this.has(url.href)) {
+      const found = this.get(url.href);
+      if (found) {
+        return found;
+      }
+    }
+
+    const mimeType = mime.lookup(fileUrl.pathname);
+    const skip = this.options.skips && this.options.skips.some(skip => minimatch(url.href, skip));
+
+    /** @type {keyof classMap} */
+    let typeClass = 'Asset';
+    if (mimeType === 'text/html') {
+      typeClass = 'HtmlPage';
+    }
+    const asset = new classMap[typeClass](url, {
+      assetManager: this,
+      localPath,
+      originPath: this.options.originPath,
+      originUrl: this.options.originUrl,
+      onParseElementCallbacks: this.options.onParseElementCallbacks,
+      skip,
+    });
+    asset.status = ASSET_STATUS.existsLocal;
+    this.assets.set(this.normalizeUrl(url.href), asset);
+
+    for (const plugin of this.options.plugins) {
+      plugin.onNewParsedAsset(asset);
+    }
+
+    return asset;
+  }
+
+  /**
+   * @param {URL | string} url
+   * @param {{ mimeType?: string }} options
+   * @returns {Asset | HtmlPage}
+   */
+  addUrl(url, { mimeType = '' } = {}) {
+    const useUrl = typeof url === 'string' ? getUrl(url) : url;
+    if (this.has(useUrl.href)) {
+      return /** @type {Asset | HtmlPage} */ (this.get(useUrl.href));
+    }
+
+    const skip =
+      this.options.skips && this.options.skips.some(skip => minimatch(useUrl.href, skip));
+
+    /** @type {keyof classMap} */
+    let typeClass = 'Asset';
+    if (mimeType === 'text/html') {
+      typeClass = 'HtmlPage';
+    }
+    const asset = new classMap[typeClass](useUrl, {
+      assetManager: this,
+      originPath: this.options.originPath,
+      originUrl: this.options.originUrl,
+      onParseElementCallbacks: this.options.onParseElementCallbacks,
+      skip,
+    });
+
+    this.assets.set(this.normalizeUrl(useUrl.href), asset);
+
+    for (const plugin of this.options.plugins) {
+      plugin.onNewParsedAsset(asset);
+    }
+
+    return asset;
+  }
+
+  /**
+   * @param {string} url
+   * @returns {string}
+   */
+  normalizeUrl(url) {
+    if (this.options.isLocalUrl(url)) {
+      return normalizeToLocalUrl(url);
+    }
+    return normalizeUrl(url);
+  }
+
+  /**
+   * @param {string} url
+   * @returns
+   */
+  get(url) {
+    return this.assets.get(this.normalizeUrl(url));
+  }
+
+  /**
+   * @param {string} url
+   * @returns {boolean}
+   */
+  has(url) {
+    return this.assets.has(this.normalizeUrl(url));
+  }
+
+  get size() {
+    return this.assets.size;
+  }
+
+  /**
+   * @param {string} url
+   * @returns {Asset | HtmlPage}
+   */
+  getAsset(url) {
+    let asset = this.get(url);
+    if (!asset) {
+      asset = this.addUrl(new URL(url));
+    }
+    return asset;
+  }
+
+  getAll() {
+    return Array.from(this.assets.values());
+  }
+
+  get isIdle() {
+    return this.fetchQueue.isIdle && this.parsingQueue.isIdle;
+  }
+}

packages/check-website/src/assets/HtmlPage.js

@@ -0,0 +1,246 @@
+import fs from 'fs';
+
+import { Asset, ASSET_STATUS } from './Asset.js';
+
+import {
+  getAttributeInfo,
+  getLinksFromSrcSet,
+  resolveToFullPageUrl,
+} from '../helpers/sax-helpers.js';
+import { parser, SaxEventType, streamOptions } from '../helpers/sax-parser.js';
+
+/** @typedef {import('sax-wasm').Tag} Tag */
+/** @typedef {import('../../types/main.js').Reference} Reference */
+
+export class HtmlPage extends Asset {
+  /**
+   * @type {Reference[]}
+   */
+  references = [];
+
+  /**
+   * A list of hashes that can be used as URL fragments to jump to specific parts of the page.
+   * @type {string[]}
+   */
+  hashes = [];
+
+  /**
+   * The target URL of a `<meta http-equiv="refresh" content="...">` element
+   * contained on the page (if any).
+   * @type {URL | undefined}
+   */
+  redirectTargetUrl;
+
+  referenceSources = [
+    { tagName: 'img', attribute: 'src' },
+    { tagName: 'img', attribute: 'srcset' },
+    { tagName: 'source', attribute: 'src' },
+    { tagName: 'source', attribute: 'srcset' },
+    { tagName: 'a', attribute: 'href' },
+    { tagName: 'link', attribute: 'href' },
+    { tagName: 'script', attribute: 'src' },
+  ];
+
+  /** @type {import('../../types/main.js').FullHtmlPageOptions} */
+  options = {
+    ...this.options,
+    onParseElementCallbacks: [],
+  };
+
+  /**
+   * @param {URL} url
+   * @param {import('../../types/main.js').HtmlPageOptions} options
+   */
+  constructor(url, options) {
+    super(url, {
+      onParseElementCallbacks: [],
+      ...options,
+    });
+    this.options = { ...this.options, ...options };
+  }
+
+  /**
+   * @param {string} hash
+   * @returns {boolean}
+   */
+  hasHash(hash) {
+    const checkHash = hash.startsWith('#') ? hash.substring(1) : hash;
+    return this.hashes.includes(checkHash);
+  }
+
+  async parse() {
+    if (!(await this.exists())) {
+      return;
+    }
+
+    return await this._parse();
+  }
+
+  #parsing = false;
+  #parsingPromise = Promise.resolve(false);
+
+  /**
+   *
+   * @protected
+   * @returns {Promise<boolean>}
+   */
+  _parse() {
+    if (this.#parsing) {
+      return this.#parsingPromise;
+    }
+
+    this.#parsing = true;
+    this.#parsingPromise = new Promise(resolve => {
+      if (this.status >= ASSET_STATUS.parsed && this.status < ASSET_STATUS.missing) {
+        this.#parsing = false;
+        resolve(true);
+      } else {
+        if (!this.options.assetManager) {
+          throw new Error('You need to pass an assetManager to the options');
+        }
+        this.options.assetManager?.parsingQueue
+          .add(async () => await this.executeParse())
+          .then(() => {
+            this.#parsing = false;
+            resolve(true);
+          });
+      }
+    });
+    return this.#parsingPromise;
+  }
+
+  async executeParse() {
+    if (this.options.skip) {
+      return;
+    }
+    if (!(await this.exists())) {
+      return;
+    }
+    await this._executeParse();
+  }
+
+  /**
+   *
+   * @protected
+   * @returns {Promise<void>}
+   */
+  _executeParse() {
+    parser.eventHandler = (ev, _data) => {
+      if (ev === SaxEventType.CloseTag) {
+        const data = /** @type {Tag} */ (/** @type {any} */ (_data));
+
+        const searchTags = this.referenceSources.map(({ tagName }) => tagName);
+        if (searchTags.includes(data.name)) {
+          const possibleAttributes = this.referenceSources
+            .map(({ attribute, tagName }) => (tagName === data.name ? attribute : undefined))
+            .filter(Boolean);
+          for (const possibleAttributeName of possibleAttributes) {
+            if (possibleAttributeName) {
+              const attribute = getAttributeInfo(data, possibleAttributeName);
+              if (attribute) {
+                const { value, start, end, name } = attribute;
+                /** @type {Reference} */
+                const entry = {
+                  start,
+                  end,
+                  value,
+                  url: resolveToFullPageUrl(this.url.href, value),
+                  page: this,
+                  attribute: name,
+                  tag: data.name,
+                };
+                if (name === 'srcset') {
+                  const links = getLinksFromSrcSet(value, this.url.href, entry);
+                  this.references.push(...links);
+                } else {
+                  this.references.push(entry);
+                  if (this.status === ASSET_STATUS.existsLocal) {
+                    // only add "sub" assets for local files
+                    this.options.assetManager?.addUrl(entry.url);
+                  }
+                }
+              }
+            }
+          }
+        }
+
+        const idData = getAttributeInfo(data, 'id');
+        if (idData) {
+          this.hashes.push(idData.value);
+        }
+
+        if (data.name === 'a') {
+          const nameData = getAttributeInfo(data, 'name');
+          if (nameData) {
+            this.hashes.push(nameData.value);
+          }
+        }
+
+        /** @type {import('../../types/main.js').ParseElement} */
+        const element = {
+          tagName: data.name.toUpperCase(),
+          getAttribute: name => getAttributeInfo(data, name)?.value,
+        };
+        this.options.onParseElementCallbacks.forEach(cb => cb(element, this));
+
+        // Check if the page redirects somewhere else using meta refresh
+        if (data.name === 'meta') {
+          const httpEquivData = getAttributeInfo(data, 'http-equiv');
+          if (httpEquivData && httpEquivData.value.toLowerCase() === 'refresh') {
+            const metaRefreshContent = getAttributeInfo(data, 'content')?.value;
+            const metaRefreshMatches = metaRefreshContent?.match(
+              /^([0-9]+)\s*;\s*url\s*=\s*(.+)$/i,
+            );
+            this.redirectTargetUrl = metaRefreshMatches
+              ? new URL(metaRefreshMatches[2], this.url.href)
+              : undefined;
+
+            if (this.status === ASSET_STATUS.existsLocal && this.redirectTargetUrl) {
+              // only add "sub" assets for local files
+              this.options.assetManager?.addUrl(this.redirectTargetUrl.href, {
+                mimeType: 'text/html',
+              });
+            }
+          }
+        }
+      }
+    };
+
+    return new Promise((resolve, reject) => {
+      if (this.status === ASSET_STATUS.existsLocal) {
+        if (!this.options.localPath) {
+          throw new Error(`Missing local path on the asset ${this.url.href}`);
+        }
+        // Read from FileSystem
+        const readable = fs.createReadStream(this.options.localPath, streamOptions);
+        readable.on('data', chunk => {
+          // @ts-expect-error
+          parser.write(chunk);
+        });
+        readable.on('end', () => {
+          parser.end();
+          this.status = ASSET_STATUS.parsedLocal;
+          resolve();
+        });
+      }
+
+      if (this.status === ASSET_STATUS.existsExternal) {
+        // Fetch from the network
+        this.options.fetch(this.url.href).then(async response => {
+          if (!response.ok || !response.body) {
+            reject('Error in response');
+            return;
+          }
+          response.body.on('data', chunk => {
+            parser.write(chunk);
+          });
+          response.body.on('end', () => {
+            parser.end();
+            this.status = ASSET_STATUS.parsedExternal;
+            resolve();
+          });
+        });
+      }
+    });
+  }
+}

packages/check-website/src/cli.js

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+
+import { existsSync } from 'fs';
+import path from 'path';
+import { CheckWebsiteCli } from './CheckWebsiteCli.js';
+
+const cli = new CheckWebsiteCli();
+
+const cwd = process.cwd();
+const configFiles = [
+  path.join('config', 'check-website.config.js'),
+  path.join('config', 'check-website.config.mjs'),
+  'check-website.config.js',
+  'check-website.config.mjs',
+  path.join('..', 'config', 'check-website.config.js'),
+  path.join('..', 'config', 'check-website.config.mjs'),
+  path.join('..', 'check-website.config.js'),
+  path.join('..', 'check-website.config.mjs'),
+];
+
+for (const configFile of configFiles) {
+  const configFilePath = path.join(cwd, configFile);
+  if (existsSync(configFilePath)) {
+    cli.options.configFile = configFilePath;
+    break;
+  }
+}
+
+await cli.execute();

packages/check-website/src/cli/LitTerminal.js

@@ -0,0 +1,383 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment */
+import { hideCursor } from './helpers.js';
+
+const terminal = process.stderr;
+
+export class LitTerminal {
+  dynamicRender = true;
+
+  /**
+   * They dynamic cli view.
+   * It can only render as many lines as the terminal height which is usually around 24-40.
+   * If you write more dynamic lines, an error will be thrown.
+   * To display more information you can use logStatic() to render lines that will not by dynamic.
+   *
+   * Why?
+   * If you write more lines than the terminal height then scrolling will happen and at this point it is
+   * no longer accessible by the terminal itself.
+   * Scrolling is a feature of terminal simulators, not the terminal itself.
+   * This means that once content got scrolled out of the terminal, it is no longer adjustable.
+   *
+   * @example
+   *   render() {
+   *     return cli`
+   *       Counter: ${this.counter}
+   *     `;
+   *   }
+   *
+   * @returns {string}
+   */
+  render() {
+    return '';
+  }
+
+  constructor() {
+    /** @type {typeof LitTerminal} */ (this.constructor).__finalize();
+  }
+
+  /**
+   * Log a static string that is rendered above the dynamic view.
+   *
+   * Use it to display information like a list of errors or completed tasks.
+   * This content will not be cleared when the dynamic view is rendered and will always be reachable via scrolling.
+   *
+   * @param {string} message
+   */
+  logStatic(message) {
+    this.#staticLogs.push(message);
+    this.requestUpdate();
+  }
+
+  /**
+   * Request a rerender of the dynamic view.
+   * You can call this as often as you want and it will only rerender once per LitTerminal.renderInterval.
+   * Typically you don't need to call this manually as it is called automatically when you update a property or use logStatic().
+   * You can use it to create your own getters/setters
+   * @example
+   *   _counter = 0;
+   *   set counter(counter) {
+   *     this._counter = counter;
+   *     this.requestUpdate();
+   *   }
+   *   get counter() {
+   *     return this._counter;
+   *   }s
+   */
+  requestUpdate() {
+    if (this.#updateRequested === false) {
+      this.#updateRequested = true;
+      setTimeout(() => {
+        this._render();
+        this.#updateRequested = false;
+      }, /** @type {typeof LitTerminal} */ (this.constructor).renderInterval);
+    }
+  }
+
+  /**
+   * This actually
+   */
+  execute() {
+    hideCursor();
+  }
+
+  //
+  // End of public API
+  //
+
+  #lastRender = '';
+
+  /** @type {string[]} */
+  #staticLogs = [];
+
+  #updateRequested = false;
+
+  /**
+   * How often (in ms) the dynamic view should be rerendered [defaults to 100]
+   */
+  static renderInterval = 100;
+
+  /**
+   * This writes the result of render() & logStatic() to the terminal.
+   * It compares the last render with the current render and only clears and writes the changed lines.
+   *
+   * It throws an error you render() tries to write more lines than the terminal height.
+   */
+  _render() {
+    if (this.dynamicRender === false) {
+      for (const staticLog of this.#staticLogs) {
+        console.log(staticLog);
+      }
+      this.#staticLogs = [];
+      return;
+    }
+
+    const render = this.render();
+    const renderLines = render.split('\n');
+
+    const windowSize = terminal.getWindowSize();
+    if (renderLines.length > windowSize[1]) {
+      throw new Error(
+        `You rendered ${renderLines.length} lines while the terminal height is only ${windowSize[1]}. For non dynamic parts use logStatic()`,
+      );
+    }
+
+    if (render !== this.#lastRender || this.#staticLogs.length) {
+      const lastRenderLines = this.#lastRender.split('\n');
+      if (lastRenderLines.length > 0) {
+        terminal.moveCursor(0, lastRenderLines.length * -1);
+        terminal.cursorTo(0);
+      }
+
+      const staticLength = this.#staticLogs.length;
+      if (staticLength) {
+        for (const staticLog of this.#staticLogs) {
+          terminal.clearLine(0);
+          console.log(staticLog);
+        }
+        this.#staticLogs = [];
+      }
+
+      for (const [index, line] of renderLines.entries()) {
+        if (line.length !== lastRenderLines[index - staticLength]?.length) {
+          terminal.clearLine(0);
+        }
+        console.log(line);
+      }
+      terminal.clearScreenDown();
+
+      this.#lastRender = render;
+    }
+  }
+
+  //
+  // ***********************************************************************************************
+  // Below if inspired by ReactiveElement
+  // https://github.com/lit/lit/blob/main/packages/reactive-element/src/reactive-element.ts
+  //
+
+  static finalized = false;
+  static properties = {};
+
+  /**
+   * Creates property accessors for registered properties, sets up element
+   * styling, and ensures any superclasses are also finalized. Returns true if
+   * the element was finalized.
+   * @nocollapse
+   */
+  static __finalize() {
+    if (this.finalized) {
+      return false;
+    }
+    this.finalized = true;
+    // // finalize any superclasses
+    // const superCtor = Object.getPrototypeOf(this);
+    // superCtor.finalize();
+
+    if (this.properties) {
+      const props = this.properties;
+      // support symbols in properties (IE11 does not support this)
+      const propKeys = [
+        ...Object.getOwnPropertyNames(props),
+        ...Object.getOwnPropertySymbols(props),
+      ];
+      for (const p of propKeys) {
+        // @ts-ignore
+        this.createProperty(p, props[p]);
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Creates a property accessor on the element prototype if one does not exist
+   * and stores a {@linkcode PropertyDeclaration} for the property with the
+   * given options. The property setter calls the property's `hasChanged`
+   * property option or uses a strict identity check to determine whether or not
+   * to request an update.
+   *
+   * This method may be overridden to customize properties; however,
+   * when doing so, it's important to call `super.createProperty` to ensure
+   * the property is setup correctly. This method calls
+   * `getPropertyDescriptor` internally to get a descriptor to install.
+   * To customize what properties do when they are get or set, override
+   * `getPropertyDescriptor`. To customize the options for a property,
+   * implement `createProperty` like this:
+   *
+   * ```ts
+   * static createProperty(name, options) {
+   *   options = Object.assign(options, {myOption: true});
+   *   super.createProperty(name, options);
+   * }
+   * ```
+   *
+   * @nocollapse
+   * @category properties
+   * @param {PropertyKey} name Name of property
+   * @param {import('../../types/LitTerminal.js').PropertyDeclaration} options Property declaration
+   */
+  static createProperty(name, options) {
+    // Note, since this can be called by the `@property` decorator which
+    // is called before `finalize`, we ensure finalization has been kicked off.
+    this.__finalize();
+
+    this.elementProperties.set(name, options);
+
+    // Do not generate an accessor if the prototype already has one, since
+    // it would be lost otherwise and that would never be the user's intention;
+    // Instead, we expect users to call `requestUpdate` themselves from
+    // user-defined accessors. Note that if the super has an accessor we will
+    // still overwrite it
+    // eslint-disable-next-line no-prototype-builtins
+    if (!options.noAccessor && !this.prototype.hasOwnProperty(name)) {
+      const key = typeof name === 'symbol' ? Symbol() : `__${name}`;
+      const descriptor = this.getPropertyDescriptor(name, key, options);
+      if (descriptor !== undefined) {
+        Object.defineProperty(this.prototype, name, descriptor);
+      }
+    }
+  }
+
+  /**
+   * Returns a property descriptor to be defined on the given named property.
+   * If no descriptor is returned, the property will not become an accessor.
+   * For example,
+   *
+   * ```ts
+   * class MyElement extends LitElement {
+   *   static getPropertyDescriptor(name, key, options) {
+   *     const defaultDescriptor =
+   *         super.getPropertyDescriptor(name, key, options);
+   *     const setter = defaultDescriptor.set;
+   *     return {
+   *       get: defaultDescriptor.get,
+   *       set(value) {
+   *         setter.call(this, value);
+   *         // custom action.
+   *       },
+   *       configurable: true,
+   *       enumerable: true
+   *     }
+   *   }
+   * }
+   * ```
+   *
+   * @nocollapse
+   * @category properties
+   * @param {PropertyKey} name
+   * @param {string | symbol} key
+   * @param {import('../../types/LitTerminal.js').PropertyDeclaration} options
+   * @returns {PropertyDescriptor | undefined}
+   */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  static getPropertyDescriptor(name, key, options) {
+    return {
+      /**
+       * @this {LitTerminal}
+       */
+      get() {
+        // @ts-ignore
+        return this[key];
+        // return (this as {[key: string]: unknown})[key as string];
+      },
+      /**
+       * @param {unknown} value
+       * @this {LitTerminal}
+       */
+      set(value) {
+        // const oldValue = this[name];
+        // @ts-ignore
+        this[key] = value;
+        this.requestUpdate();
+        // this.requestUpdate(name, oldValue, options);
+      },
+      configurable: true,
+      enumerable: true,
+    };
+  }
+
+  static elementProperties = new Map();
+
+  /**
+   * Returns the property options associated with the given property.
+   * These options are defined with a `PropertyDeclaration` via the `properties`
+   * object or the `@property` decorator and are registered in
+   * `createProperty(...)`.
+   *
+   * Note, this method should be considered "final" and not overridden. To
+   * customize the options for a given property, override
+   * {@linkcode createProperty}.
+   *
+   * @nocollapse
+   * @final
+   * @category properties
+   * @param {string | symbol} name
+   */
+  static getPropertyOptions(name) {
+    return this.elementProperties.get(name) || defaultPropertyDeclaration;
+  }
+
+  /**
+   * Returns a Promise that resolves when the element has completed updating.
+   * The Promise value is a boolean that is `true` if the element completed the
+   * update without triggering another update. The Promise result is `false` if
+   * a property was set inside `updated()`. If the Promise is rejected, an
+   * exception was thrown during the update.
+   *
+   * To await additional asynchronous work, override the `getUpdateComplete`
+   * method. For example, it is sometimes useful to await a rendered element
+   * before fulfilling this Promise. To do this, first await
+   * `super.getUpdateComplete()`, then any subsequent state.
+   *
+   * @category updates
+   * @returns {Promise<boolean>} A promise of a boolean that resolves to true if the update completed
+   *   without triggering another update.
+   */
+  get updateComplete() {
+    return new Promise(resolve => setTimeout(resolve, 100));
+    // TODO: implement actual waiting for a finished render
+    // return this.getUpdateComplete();
+  }
+
+  // /**
+  //  * Override point for the `updateComplete` promise.
+  //  *
+  //  * It is not safe to override the `updateComplete` getter directly due to a
+  //  * limitation in TypeScript which means it is not possible to call a
+  //  * superclass getter (e.g. `super.updateComplete.then(...)`) when the target
+  //  * language is ES5 (https://github.com/microsoft/TypeScript/issues/338).
+  //  * This method should be overridden instead. For example:
+  //  *
+  //  * ```ts
+  //  * class MyElement extends LitElement {
+  //  *   override async getUpdateComplete() {
+  //  *     const result = await super.getUpdateComplete();
+  //  *     await this._myChild.updateComplete;
+  //  *     return result;
+  //  *   }
+  //  * }
+  //  * ```
+  //  *
+  //  * @returns {Promise<boolean>} A promise of a boolean that resolves to true if the update completed
+  //  *     without triggering another update.
+  //  * @category updates
+  //  */
+  // getUpdateComplete() {
+  //   return this.__updatePromise;
+  // }
+}
+
+const defaultPropertyDeclaration = {
+  type: String,
+  hasChanged: notEqual,
+};
+
+/**
+ * Change function that returns true if `value` is different from `oldValue`.
+ * This method is used as the default for a property's `hasChanged` function.
+ * @param {unknown} value The new value
+ * @param {unknown} old The old value
+ */
+export function notEqual(value, old) {
+  // This ensures (old==NaN, value==NaN) always returns false
+  return old !== value && (old === old || value === value);
+}

packages/check-website/src/cli/cli.js

@@ -0,0 +1,72 @@
+// TODO: support nested lists with new lines like: ${items.map(item => cli`[ ] ${item}\n`)}
+// will most likely require a stateful "cli tagged template literal" that passes on the current indent level
+
+/**
+ * Tagged template literal that
+ * - dedents the string
+ * - removes empty first/last lines
+ * - joins arrays
+ *
+ * @example
+ *   const str = cli`
+ *     Welcome ${name}!
+ *     List: ${items.map(item => `[ ] ${item} `)}
+ *   `;
+ *   # becomes
+ *   Welcome John!
+ *   List: [ ] a [ ] b [ ] c
+ *
+ * @param {TemplateStringsArray} strings
+ * @param  {...any} values
+ * @returns
+ */
+export function cli(strings, ...values) {
+  const useStrings = typeof strings === 'string' ? [strings] : strings.raw;
+  let result = '';
+
+  for (var i = 0; i < useStrings.length; i++) {
+    let currentString = useStrings[i];
+    currentString
+      // join lines when there is a suppressed newline
+      .replace(/\\\n[ \t]*/g, '')
+      // handle escaped backticks
+      .replace(/\\`/g, '`');
+    result += currentString;
+
+    if (i < values.length) {
+      const value = Array.isArray(values[i]) ? values[i].join('') : values[i];
+      result += value;
+    }
+  }
+
+  // now strip indentation
+  const lines = result.split('\n');
+  let minIndent = -1;
+  for (const line of lines) {
+    const match = line.match(/^(\s+)\S+/);
+    if (match) {
+      const indent = match[1].length;
+      if (minIndent === -1) {
+        // this is the first indented line
+        minIndent = indent;
+      } else {
+        minIndent = Math.min(minIndent, indent);
+      }
+    }
+  }
+
+  let finalResult = '';
+  if (minIndent !== -1) {
+    for (const line of lines) {
+      finalResult += line[0] === ' ' ? line.slice(minIndent) : line;
+      finalResult += '\n';
+    }
+  }
+
+  return (
+    finalResult
+      .trim()
+      // handle escaped newlines at the end to ensure they don't get stripped too
+      .replace(/\\n/g, '\n')
+  );
+}

packages/check-website/src/cli/helpers.js

@@ -0,0 +1,11 @@
+export function hr(length = process.stderr.getWindowSize()[0]) {
+  return '─'.repeat(length);
+}
+
+export function hideCursor() {
+  process.stderr.write('\u001B[?25l');
+}
+
+export function showCursor() {
+  process.stderr.write('\u001B[?25h');
+}

packages/check-website/src/cli/renderProgressBar.js

@@ -0,0 +1,40 @@
+import { gray, white } from 'colorette';
+
+const PROGRESS_BLOCKS = [' ', '▏', '▎', '▍', '▌', '▋', '▊', '▉', '█'];
+const PROGRESS_WIDTH = 30;
+
+/**
+ *
+ * @param {number} value
+ * @param {number} total
+ * @returns
+ */
+function createProgressBlocks(value, total) {
+  if (value >= total) {
+    return PROGRESS_BLOCKS[8].repeat(PROGRESS_WIDTH);
+  }
+
+  const count = (PROGRESS_WIDTH * value) / total;
+  const floored = Math.floor(count);
+  const partialBlock =
+    PROGRESS_BLOCKS[Math.floor((count - floored) * (PROGRESS_BLOCKS.length - 1))];
+  return `${PROGRESS_BLOCKS[8].repeat(floored)}${partialBlock}${' '.repeat(
+    PROGRESS_WIDTH - floored - 1,
+  )}`;
+}
+
+/**
+ *
+ * @param {number} finished
+ * @param {number} active
+ * @param {number} total
+ * @returns
+ */
+export function renderProgressBar(finished, active, total) {
+  const progressBlocks = createProgressBlocks(finished + active, total);
+  const finishedBlockCount = Math.floor((PROGRESS_WIDTH * finished) / total);
+
+  const finishedBlocks = white(progressBlocks.slice(0, finishedBlockCount));
+  const scheduledBlocks = gray(progressBlocks.slice(finishedBlockCount));
+  return `|${finishedBlocks}${scheduledBlocks}|`;
+}

packages/check-website/src/helpers/Queue.js

@@ -0,0 +1,50 @@
+import { formatPerformance } from './formatPerformance.js';
+import PQueue from 'p-queue';
+
+/** @typedef {import('p-queue').QueueAddOptions} QueueAddOptions */
+/** @typedef {import('p-queue').PriorityQueue} PriorityQueue  */
+
+export class Queue extends PQueue {
+  #total = 0;
+
+  /** @type {[number, number] | undefined} */
+  #durationStart;
+  /** @type {[number, number] | undefined} */
+  duration;
+
+  isIdle = true;
+
+  /**
+   * @param {import('p-queue').Options<PriorityQueue, QueueAddOptions>} [options]
+   */
+  constructor(options) {
+    super(options);
+    this.on('active', () => {
+      if (!this.#durationStart) {
+        this.#durationStart = process.hrtime();
+      }
+      this.isIdle = false;
+    });
+    this.on('completed', () => {
+      this.duration = process.hrtime(this.#durationStart);
+    });
+    this.on('idle', () => {
+      this.isIdle = true;
+    });
+    this.on('add', () => {
+      this.#total += 1;
+    });
+  }
+
+  getDuration() {
+    return this.duration ? formatPerformance(this.duration) : '0.00';
+  }
+
+  getDone() {
+    return this.#total - this.size - this.pending;
+  }
+
+  getTotal() {
+    return this.#total;
+  }
+}

packages/check-website/src/helpers/decodeNumberHtmlEntities.js

@@ -0,0 +1,15 @@
+/**
+ * Converts number HTML entities to their corresponding characters.
+ *
+ * Example:
+ * mailto: => mailto:
+ *
+ * @param {string} str
+ * @returns {string}
+ */
+export function decodeNumberHtmlEntities(str) {
+  return str.replace(/&#([0-9]{1,3});/gi, (match, numStr) => {
+    const num = parseInt(numStr, 10); // read num as normal number
+    return String.fromCharCode(num);
+  });
+}

packages/check-website/src/helpers/formatPerformance.js

@@ -0,0 +1,7 @@
+/**
+ * @param {[number, number]} perf
+ * @returns {string}
+ */
+export function formatPerformance(perf) {
+  return (perf[0] + perf[1] / 1e9).toFixed(2);
+}

packages/check-website/src/helpers/gatherFiles.js

@@ -0,0 +1,45 @@
+import { readdir } from 'fs/promises';
+import path from 'path';
+
+/**
+ * @param {string} fileName
+ * @returns {boolean}
+ */
+function isIndexFile(fileName) {
+  return fileName === 'index.html';
+}
+
+/**
+ * @param {string | URL} inRootDir
+ * @returns {Promise<string[]>}
+ */
+export async function gatherFiles(inRootDir) {
+  const rootDir = inRootDir instanceof URL ? inRootDir.pathname : path.resolve(inRootDir);
+  let files = [];
+
+  const entries = await readdir(rootDir, { withFileTypes: true });
+
+  // 1. handle possible index.html file
+  for (const entry of entries) {
+    if (!entry.isDirectory() && isIndexFile(entry.name)) {
+      files.push(path.join(rootDir, entry.name));
+    }
+  }
+  // 2. handle other html files
+  for (const entry of entries) {
+    const { name } = entry;
+    if (entry.isFile() && !isIndexFile(name)) {
+      const filePath = path.join(rootDir, name);
+      files.push(filePath);
+    }
+  }
+  // 3. handle sub directories
+  for (const entry of entries) {
+    if (entry.isDirectory()) {
+      const currentPath = path.join(rootDir, entry.name);
+      files.push(...(await gatherFiles(currentPath)));
+    }
+  }
+
+  return files;
+}

packages/check-website/src/helpers/normalizeUrl.js

@@ -0,0 +1,42 @@
+import normalizeUrlDep from 'normalize-url';
+
+const normalizeOptions = {
+  stripAuthentication: true,
+  stripHash: true,
+  stripTextFragment: true,
+  removeQueryParameters: false,
+  // removeTrailingSlash: false,
+  // removeSingleSlash: false,
+  removeDirectoryIndex: true,
+  // removeExplicitPort: true,
+};
+
+/**
+ * @param {string} url
+ * @param {import('normalize-url').Options} options
+ * @returns {string}
+ */
+export function normalizeUrl(url, options = {}) {
+  if (url.startsWith('mailto:')) {
+    return url.toLowerCase();
+  }
+  // = "mailto:" but html encoded)
+  if (url.startsWith('mailto:')) {
+    return url;
+  }
+  if (url.startsWith('tel:')) {
+    return url.toLowerCase();
+  }
+  if (url.startsWith('about:')) {
+    return url.toLowerCase();
+  }
+  return normalizeUrlDep(url, { ...normalizeOptions, ...options }).toLowerCase();
+}
+
+/**
+ * @param {string} url
+ * @returns {string}
+ */
+export function normalizeToLocalUrl(url) {
+  return normalizeUrl(url, { removeQueryParameters: true });
+}

packages/check-website/src/helpers/sax-helpers.js

@@ -0,0 +1,66 @@
+/** @typedef {import('sax-wasm').Tag} Tag */
+
+/**
+ * @param {Tag} data
+ * @param {string} name
+ */
+export function getAttributeInfo(data, name) {
+  if (data.attributes) {
+    const { attributes } = data;
+    const foundIndex = attributes.findIndex(entry => entry.name.value === name);
+    if (foundIndex !== -1) {
+      const entry = attributes[foundIndex].value;
+      return {
+        value: entry.value,
+        start: `${entry.start.line + 1}:${entry.start.character}`,
+        end: `${entry.end.line + 1}:${entry.end.character}`,
+        name,
+      };
+    }
+  }
+  return undefined;
+}
+
+/**
+ * @param {string} pageUrl
+ * @param {string} referenceUrl
+ * @returns {string}
+ */
+export function resolveToFullPageUrl(pageUrl, referenceUrl) {
+  // = "mailto:" but html encoded)
+  if (referenceUrl.startsWith('mailto:')) {
+    return referenceUrl;
+  }
+  if (referenceUrl.startsWith('about:')) {
+    return referenceUrl;
+  }
+  const url = new URL(referenceUrl, pageUrl);
+  return url.href;
+}
+
+/**
+ * @param {string} value Comma
+ * @param {string} pageUrl
+ * @param {import('../../types/main.js').Reference} entry
+ * @returns
+ */
+export function getLinksFromSrcSet(value, pageUrl, entry) {
+  const links = [];
+  if (value.includes(',')) {
+    const srcsets = value.split(',').map(el => el.trim());
+    for (const srcset of srcsets) {
+      if (srcset.includes(' ')) {
+        const srcsetParts = srcset.split(' ');
+        links.push({ ...entry, url: resolveToFullPageUrl(pageUrl, srcsetParts[0]) });
+      } else {
+        links.push({ ...entry, url: resolveToFullPageUrl(pageUrl, srcset) });
+      }
+    }
+  } else if (value.includes(' ')) {
+    const srcsetParts = value.split(' ');
+    links.push({ ...entry, url: resolveToFullPageUrl(pageUrl, srcsetParts[0]) });
+  } else {
+    links.push(entry);
+  }
+  return links;
+}

packages/check-website/src/helpers/sax-parser.js

@@ -0,0 +1,14 @@
+import saxWasm from 'sax-wasm';
+import { createRequire } from 'module';
+import { readFile } from 'fs/promises';
+
+export const { SaxEventType, SAXParser } = saxWasm;
+
+const require = createRequire(import.meta.url);
+
+export const streamOptions = { highWaterMark: 256 * 1024 };
+const saxPath = require.resolve('sax-wasm/lib/sax-wasm.wasm');
+const saxWasmBuffer = await readFile(saxPath);
+export const parser = new SAXParser(SaxEventType.CloseTag, streamOptions);
+
+await parser.prepareWasm(saxWasmBuffer);

packages/check-website/src/index.js

@@ -0,0 +1,14 @@
+export { normalizeToLocalUrl, normalizeUrl } from './helpers/normalizeUrl.js';
+
+export { CheckWebsiteCli } from './CheckWebsiteCli.js';
+
+export { LocalReferencesPlugin } from './plugins/LocalReferencesPlugin.js';
+export { ExternalReferencesPlugin } from './plugins/ExternalReferencesPlugin.js';
+export { HasCanonicalPlugin } from './plugins/HasCanonicalPlugin.js';
+
+export { Asset, ASSET_STATUS } from './assets/Asset.js';
+export { HtmlPage } from './assets/HtmlPage.js';
+export { AssetManager } from './assets/AssetManager.js';
+
+
+/** @typedef {import('../types/main.js').CheckWebsiteCliOptions} CheckWebsiteCliOptions */

packages/check-website/src/issues/Issue.js

@@ -0,0 +1,33 @@
+export class Issue {
+  options = {
+    sortOrder: 0,
+    duplicate: false,
+    filePath: '',
+    title: 'Issue',
+    message: '',
+    icon: '❌',
+    logger: console.log,
+    page: null,
+    skip: false,
+  };
+
+  /**
+   * @param {Partial<{}>} options
+   */
+  constructor(options = {}) {
+    this.options = { ...this.options, ...options };
+  }
+
+  /**
+   * @param {(msg: string) => void} logger
+   * @returns {void}
+   */
+  render(logger) {
+    const useLogger = logger || this.options.logger;
+    if (this.options.duplicate) {
+      return;
+    }
+    useLogger(`${this.options.icon} ${this.options.title}: ${this.options.message}`);
+    useLogger(`  🛠️  ${this.options.filePath}`);
+  }
+}

packages/check-website/src/issues/IssueManager.js

@@ -0,0 +1,24 @@
+export class IssueManager {
+  renderErrorsOnAdd = true;
+  logger = console.log;
+
+  /**
+   * @type {(import('./Issue.js').Issue | import('./PageIssue.js').PageIssue | import('./ReferenceIssue.js').ReferenceIssue)[]}
+   */
+  issues = [];
+
+  /**
+   *
+   * @param {import('./Issue.js').Issue | import('./PageIssue.js').PageIssue | import('./ReferenceIssue.js').ReferenceIssue} issue
+   */
+  add(issue) {
+    this.issues.push(issue);
+    if (this.renderErrorsOnAdd) {
+      issue.render(this.logger);
+    }
+  }
+
+  all() {
+    return this.issues;
+  }
+}

packages/check-website/src/issues/PageIssue.js

@@ -0,0 +1,11 @@
+import { Issue } from './Issue.js';
+
+export class PageIssue extends Issue {
+  constructor(options = {}) {
+    super({
+      message: '',
+      title: 'Page Issue',
+      ...options,
+    });
+  }
+}

packages/check-website/src/issues/ReferenceIssue.js

@@ -0,0 +1,11 @@
+import { Issue } from './Issue.js';
+
+export class ReferenceIssue extends Issue {
+  constructor(options = {}) {
+    super({
+      title: 'Not Found',
+      icon: '🔗',
+      ...options,
+    });
+  }
+}

packages/check-website/src/plugins/ExternalReferencesPlugin.js

@@ -0,0 +1,77 @@
+import path from 'path';
+import { cli } from '../cli/cli.js';
+import { ReferenceIssue } from '../issues/ReferenceIssue.js';
+import { Plugin } from './Plugin.js';
+
+/** @typedef {import('../assets/HtmlPage.js').HtmlPage} HtmlPage */
+/** @typedef {import('../../types/main.js').Reference} Reference */
+/** @typedef {import('../../types/main.js').CheckContext} CheckContext */
+/** @typedef {import('../../types/main.js').AddToQueueHelpers} AddToQueueHelpers */
+
+/**
+ * @param {string} url
+ */
+function getDomain(url) {
+  try {
+    const { hostname } = new URL(url);
+    return hostname;
+  } catch {
+    return '';
+  }
+}
+
+export class ExternalReferencesPlugin extends Plugin {
+  domainStats = new Map();
+
+  constructor(options = {}) {
+    super({
+      title: 'External',
+      checkLabel: 'links',
+      ...options,
+    });
+  }
+
+  /**
+   * @param {CheckContext} context
+   */
+  async check(context) {
+    const { item, report, getAsset } = context;
+    const reference = /** @type {Reference} */ (item);
+    const asset = getAsset(reference.url);
+
+    if (!(await asset.exists())) {
+      const { page } = reference;
+      const relPath = path.relative(process.cwd(), page.localPath);
+      const filePath = `./${relPath}:${reference.start}`;
+      const message = `<${reference.tag} ${reference.attribute}="${reference.value}">`;
+      report(new ReferenceIssue({ page, filePath, message }));
+    }
+  }
+
+  /**
+   * @param {HtmlPage} page
+   * @param {AddToQueueHelpers} helpers
+   * @returns {Promise<Reference[]>}
+   */
+  async addToQueue(page, helpers) {
+    const { isLocalUrl } = helpers;
+    const checkItems = [];
+    for (const reference of page.references) {
+      if (reference.url.startsWith('http') && !isLocalUrl(reference.url)) {
+        checkItems.push(reference);
+        const domain = getDomain(reference.url);
+        this.domainStats.set(domain, (this.domainStats.get(domain) || 0) + 1);
+      }
+    }
+    return checkItems;
+  }
+
+  render() {
+    const top3 = [...this.domainStats.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3);
+    const top3Str = top3.map(([domain, count], i) => `${i + 1}. ${domain} (${count})`).join(', ');
+    return cli`
+      ${super.render()}
+      - Top Domains: ${top3Str}
+    `;
+  }
+}

packages/check-website/src/plugins/HasCanonicalPlugin.js

@@ -0,0 +1,60 @@
+import path from 'path';
+import { PageIssue } from '../issues/PageIssue.js';
+import { Plugin } from './Plugin.js';
+
+/** @typedef {import('../assets/HtmlPage.js').HtmlPage} HtmlPage */
+/** @typedef {import('../../types/main.js').CheckContext} CheckContext */
+/** @typedef {import('../../types/main.js').Reference} Reference */
+/** @typedef {import('../../types/main.js').ParseElement} ParseElement */
+
+export class HasCanonicalPlugin extends Plugin {
+  /**
+   * Contains the unique absolute page URL as declared by the
+   * `<link rel="canonical" href="...">` element (if any) for every page.
+   *
+   * @type {Map<HtmlPage, URL>}
+   */
+  canonicalUrls = new Map();
+
+  /**
+   * @param {ParseElement} element
+   * @param {HtmlPage} page
+   */
+  onParseElement = (element, page) => {
+    if (!this.canonicalUrls.has(page) && element.tagName === 'LINK') {
+      if (element.getAttribute('rel') === 'canonical') {
+        const href = element.getAttribute('href');
+        if (href) {
+          this.canonicalUrls.set(page, new URL(href));
+        }
+      }
+    }
+  };
+
+  constructor(options = {}) {
+    super({
+      title: 'Canonical',
+      checkLabel: 'pages',
+      ...options,
+    });
+  }
+
+  /**
+   * @param {CheckContext} context
+   */
+  async check(context) {
+    const page = /** @type {HtmlPage} */ (context.item);
+    if (!this.canonicalUrls.has(page)) {
+      context.report(
+        new PageIssue({
+          title: 'Missing canonical',
+          message: 'The page is missing a <link rel="canonical" href="...">',
+          page,
+          filePath: './' + path.relative(process.cwd(), page.localPath),
+          icon: '🦄',
+        }),
+      );
+    }
+    return;
+  }
+}

packages/check-website/src/plugins/LocalReferencesPlugin.js

@@ -0,0 +1,68 @@
+import normalizeUrl from 'normalize-url';
+import path from 'path';
+import { HtmlPage } from '../assets/HtmlPage.js';
+import { ReferenceIssue } from '../issues/ReferenceIssue.js';
+import { Plugin } from './Plugin.js';
+
+/** @typedef {import('../../types/main.js').Reference} Reference */
+/** @typedef {import('../../types/main.js').CheckContext} CheckContext */
+/** @typedef {import('../../types/main.js').AddToQueueHelpers} AddToQueueHelpers */
+/** @typedef {import('../../types/main.js').PluginInterface} PluginInterface */
+
+/**
+ * @implement {PluginInterface}
+ */
+export class LocalReferencesPlugin extends Plugin {
+  constructor(options = {}) {
+    super({
+      title: 'Local',
+      checkLabel: 'links',
+      ...options,
+    });
+  }
+
+  /**
+   * @param {HtmlPage} page
+   * @param {AddToQueueHelpers} helpers
+   * @returns {Promise<Reference[]>}
+   */
+  async addToQueue(page, helpers) {
+    const { isLocalUrl } = helpers;
+    const checkItems = [];
+    for (const reference of page.references) {
+      if (isLocalUrl(reference.url)) {
+        checkItems.push(reference);
+      }
+    }
+    return checkItems;
+  }
+
+  /**
+   * @param {CheckContext} context
+   */
+  async check(context) {
+    const { item, report, getAsset } = context;
+    const reference = /** @type {Reference} */ (item);
+    const targetAsset = getAsset(reference.url);
+
+    const { page } = reference;
+    const relPath = path.relative(process.cwd(), page.localPath);
+    const filePath = `./${relPath}:${reference.start}`;
+    const message = `<${reference.tag} ${reference.attribute}="${reference.value}">`;
+
+    if (await targetAsset.exists()) {
+      const urlFocusHash = normalizeUrl(reference.url, { stripTextFragment: true }); // removes :~:text=
+      const hash = urlFocusHash.includes('#') ? urlFocusHash.split('#')[1] : '';
+      if (
+        hash &&
+        targetAsset instanceof HtmlPage &&
+        (await targetAsset.parse()) &&
+        !targetAsset.hasHash(hash)
+      ) {
+        report(new ReferenceIssue({ page, filePath, message, icon: '#️⃣' }));
+      }
+    } else {
+      report(new ReferenceIssue({ page, filePath, message }));
+    }
+  }
+}

packages/check-website/src/plugins/Plugin.js

@@ -0,0 +1,215 @@
+import { gray, green, red } from 'colorette';
+import { EventEmitter } from 'events';
+import { ASSET_STATUS } from '../assets/Asset.js';
+import { HtmlPage } from '../assets/HtmlPage.js';
+import { renderProgressBar } from '../cli/renderProgressBar.js';
+import { Queue } from '../helpers/Queue.js';
+
+/** @typedef {import('../assets/Asset.js').Asset} Asset */
+/** @typedef {import('../CheckWebsiteCli.js').CheckWebsiteCli} CheckWebsiteCli */
+
+/** @typedef {import('../../types/main.js').Reference} Reference */
+/** @typedef {import('../../types/main.js').CheckContext} CheckContext */
+/** @typedef {import('../../types/main.js').AddToQueueHelpers} AddToQueueHelpers */
+/** @typedef {import('../../types/main.js').PluginInterface} PluginInterface */
+
+export class Plugin {
+  /** @type {import('../issues/IssueManager.js').IssueManager | undefined} */
+  issueManager;
+
+  /** @type {import('../assets/AssetManager.js').AssetManager | undefined} */
+  assetManager;
+
+  /** @type {CheckWebsiteCli | undefined} */
+  cli;
+
+  _passed = 0;
+  _failed = 0;
+  _skipped = 0;
+
+  /**
+   * @type {[number, number] | undefined}
+   */
+  _performanceStart;
+
+  /**
+   * @type {Map<string, unknown>}
+   */
+  _checkItems = new Map();
+
+  _queue = new Queue();
+
+  _processedPages = new Set();
+
+  /**
+   * @readonly
+   */
+  events = new EventEmitter();
+
+  /**
+   * @param {Asset} asset
+   */
+  async onNewParsedAsset(asset) {
+    if (asset instanceof HtmlPage) {
+      asset.events.on('status-changed', async () => {
+        if (asset.status >= ASSET_STATUS.parsed) {
+          if (!this._processedPages.has(asset)) {
+            this._processedPages.add(asset);
+            /** @type {AddToQueueHelpers} */
+            const helpers = {
+              isLocalUrl: url => this.isLocalUrl(url),
+            };
+            const newQueueItems = await this.addToQueue(asset, helpers);
+            newQueueItems.forEach(_item => {
+              this._queue.add(async () => {
+                const item = /** @type {Reference | HtmlPage} */ (_item);
+
+                let skip = false;
+                if (item.url) {
+                  const url = item.url instanceof URL ? item.url.href : item.url;
+                  const targetAsset = this.assetManager?.getAsset(url);
+                  if (this.isLocalUrl(url)) {
+                    if (targetAsset instanceof HtmlPage) {
+                      targetAsset.parse(); // no await but we request the parse => e.g. we crawl
+                    }
+                  }
+                  if (targetAsset?.options.skip) {
+                    skip = true;
+                  }
+                }
+
+                if (skip === false) {
+                  let hadIssues = false;
+                  /** @type {CheckContext} */
+                  const context = {
+                    report: issue => {
+                      hadIssues = true;
+                      this.issueManager?.add(issue);
+                    },
+                    item,
+                    getAsset: url => {
+                      if (!this.assetManager) {
+                        throw Error('Asset manager not available');
+                      }
+                      return this.assetManager.getAsset(url);
+                    },
+                    isLocalUrl: url => this.isLocalUrl(url),
+                  };
+                  await /** @type {PluginInterface} */ (/** @type {unknown} */ (this)).check(
+                    context,
+                  );
+                  if (hadIssues) {
+                    this._failed += 1;
+                  } else {
+                    this._passed += 1;
+                  }
+                } else {
+                  this._skipped += 1;
+                }
+
+                this.events.emit('progress');
+              });
+            });
+          }
+        }
+      });
+    }
+  }
+
+  /**
+   * @param {HtmlPage} page
+   * @param {AddToQueueHelpers} helpers
+   * @returns {Promise<unknown[]>}
+   */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async addToQueue(page, helpers) {
+    return [page];
+  }
+
+  /**
+   * @param {Partial<{}>} options
+   */
+  constructor(options = {}) {
+    this.options = {
+      title: 'Plugin',
+      checkLabel: 'pages',
+      ...options,
+    };
+
+    if (this.options.title.length > 10) {
+      throw new Error(`Plugin title should be max 10 characters. Given "${this.options.title}"`);
+    }
+
+    this._queue.on('idle', () => {
+      this.events.emit('idle');
+    });
+  }
+
+  get isIdle() {
+    return this._queue.isIdle;
+  }
+
+  /**
+   * @param {CheckWebsiteCli} cli
+   */
+  setup(cli) {
+    this._performanceStart = process.hrtime();
+    this.cli = cli;
+  }
+
+  getTotal() {
+    return this._queue.getTotal();
+  }
+
+  getDuration() {
+    return this._queue.getDuration();
+  }
+
+  getDone() {
+    return this._queue.getDone();
+  }
+
+  getPassed() {
+    return this._passed;
+  }
+
+  getFailed() {
+    return this._failed;
+  }
+
+  getSkipped() {
+    return this._skipped;
+  }
+
+  /**
+   * @param {string} url
+   * @returns {boolean}
+   */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  isLocalUrl(url) {
+    return true;
+  }
+
+  render() {
+    const checkLabel = this.options.checkLabel;
+    const doneNr = this.getDone();
+    const passed = this.getPassed();
+    const failed = this.getFailed();
+    const skipped = this.getSkipped();
+    const total = this.getTotal();
+
+    const title = `${this.options.title}:`.padEnd(11);
+    const progress = renderProgressBar(doneNr, 0, total);
+
+    const minNumberLength = `${total}`.length;
+    const done = `${doneNr}`.padStart(minNumberLength);
+
+    const passedTxt = passed > 0 ? `${green(`${passed} passed`)}` : '0 passed';
+    const failedTxt = failed > 0 ? `, ${red(`${failed} failed`)}` : '';
+    const skippedTxt = skipped > 0 ? `, ${gray(`${skipped} skipped`)}` : '';
+    const resultTxt = `${passedTxt}${failedTxt}${skippedTxt}`;
+    const duration = this.getDuration();
+
+    return `${title} ${progress} ${done}/${total} ${checkLabel} | 🕑 ${duration}s | ${resultTxt}`;
+  }
+}

packages/check-website/test-node/00-Asset.test.js

@@ -0,0 +1,56 @@
+import { expect } from 'chai';
+import { HtmlPage, AssetManager } from '../src/index.js';
+
+const testOptions = {
+  originUrl: 'https://example.com/',
+  originPath: new URL('.', import.meta.url).pathname,
+};
+
+describe('Asset', () => {
+  it('01: add local file via file url exists', async () => {
+    const assets = new AssetManager(testOptions);
+    const asset = assets.addExistingFile(
+      new URL('fixtures/01-AssetManager/file.txt', import.meta.url),
+    );
+    expect(await asset.exists()).to.be.true;
+  });
+
+  it('01b: local file via http url exists if added before', async () => {
+    const assets = new AssetManager(testOptions);
+    assets.addExistingFile(new URL('fixtures/01-AssetManager/file.txt', import.meta.url));
+    const asset = assets.addUrl(new URL('https://example.com/fixtures/01-AssetManager/file.txt'));
+    expect(await asset.exists()).to.be.true;
+  });
+
+  it('01c: local file missing', async () => {
+    const assets = new AssetManager(testOptions);
+    const asset = assets.addUrl(
+      new URL('https://example.com/fixtures/01-AssetManager/missing.txt'),
+    );
+    expect(await asset.exists()).to.be.false;
+  });
+
+  it('01d: local html page exists', async () => {
+    const assets = new AssetManager(testOptions);
+    const page = assets.addExistingFile(
+      new URL('fixtures/01-AssetManager/page.html', import.meta.url),
+    );
+    expect(page).to.be.an.instanceOf(HtmlPage);
+    expect(await page.exists()).to.be.true;
+  });
+
+  it('02: external file exists', async () => {
+    const assets = new AssetManager(testOptions);
+    const asset = assets.addUrl(new URL('https://rocket.modern-web.dev/favicon.ico'));
+    expect(await asset.exists()).to.be.true;
+  });
+
+  it('03: adds assets while parsing local pages', async () => {
+    const assets = new AssetManager(testOptions);
+    const page = /** @type {HtmlPage} */ (
+      assets.addExistingFile(new URL('fixtures/01-AssetManager/page.html', import.meta.url))
+    );
+    await page.parse();
+    expect(assets.size).to.equal(2);
+  });
+});

packages/check-website/test-node/01-HtmlPage.test.js

@@ -0,0 +1,452 @@
+import chai from 'chai';
+import path from 'path';
+import { Readable } from 'stream';
+import { readFile } from 'fs/promises';
+
+import { HtmlPage, ASSET_STATUS, AssetManager } from 'check-website';
+import { fileURLToPath } from 'url';
+
+class FakeReadable extends Readable {
+  _read() {
+    // noop
+  }
+}
+
+class MockedFetch {
+  constructor({ content = '' } = {}) {
+    this.stream = new FakeReadable();
+    this._content = content;
+  }
+
+  get fetch() {
+    setTimeout(async () => {
+      this.stream.push(this._content);
+      this.stream.emit('end');
+    }, 1);
+    return () => Promise.resolve({ ok: true, body: this.stream });
+  }
+
+  /**
+   * @param {any} chunk
+   */
+  push(chunk) {
+    return this.stream.push(chunk);
+  }
+}
+
+const { expect } = chai;
+
+const currentDir = path.dirname(new URL(import.meta.url).pathname);
+
+/**
+ * @param {HtmlPage} page
+ */
+function cleanup(page) {
+  const keep = {};
+  keep.hashes = page.hashes;
+  keep.redirectTargetUrl = page.redirectTargetUrl;
+  keep.references = page.references.map(ref => ({
+    url: ref.url,
+    attribute: ref.attribute,
+    tag: ref.tag,
+    value: ref.value,
+  }));
+  const optKeep = {};
+  optKeep.localPath = page.options.localPath
+    ? `abs::${path.relative(currentDir, page.options.localPath)}`
+    : page.options.localPath;
+  optKeep.localSourcePath = page.options.localSourcePath
+    ? `abs::${path.relative(currentDir, page.options.localSourcePath)}`
+    : page.options.localSourcePath;
+  keep.options = optKeep;
+  keep.url = page.url;
+  return keep;
+}
+
+/**
+ * @param {import('../types/main.js').HtmlPageOptions} options
+ */
+function withTestOptions(options) {
+  return {
+    originUrl: 'https://example.com/',
+    originPath: new URL('.', import.meta.url).pathname,
+    assetManager: new AssetManager(),
+    ...options,
+  };
+}
+
+describe('HtmlPage', () => {
+  it('01: hashes', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/01-hashes.html'),
+      withTestOptions({
+        localPath: fileURLToPath(new URL('fixtures/01-HtmlPage/01-hashes.html', import.meta.url)),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/01-hashes.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/01-hashes.html',
+        localSourcePath: '',
+      },
+      hashes: ['first', 'second', 'third'],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          value: '#',
+          url: 'https://example.com/fixtures/01-HtmlPage/01-hashes.html#',
+        },
+      ],
+    });
+  });
+
+  it.skip('01a: fetch it as an external url', async () => {
+    const mocked = new MockedFetch({
+      // @ts-ignore
+      content: await readFile(new URL('fixtures/01-HtmlPage/01-hashes.html', import.meta.url)),
+    });
+    const page = new HtmlPage(
+      new URL('https://is.mocked.com/'),
+      withTestOptions({
+        // @ts-ignore
+        fetch: mocked.fetch,
+      }),
+    );
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://is.mocked.com/'),
+      options: {
+        localPath: '',
+        localSourcePath: '',
+      },
+      hashes: ['first', 'second', 'third'],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          value: '#',
+          url: 'https://is.mocked.com/#',
+        },
+      ],
+    });
+  });
+
+  it('02: internal link', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/02-internal-link.html'),
+      withTestOptions({
+        localPath: fileURLToPath(
+          new URL('fixtures/01-HtmlPage/02-internal-link.html', import.meta.url),
+        ),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/02-internal-link.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/02-internal-link.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'https://example.com/index.html',
+          value: '/index.html',
+        },
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'https://example.com/fixtures/01-HtmlPage/index.html',
+          value: './index.html',
+        },
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'https://example.com/fixtures/01-HtmlPage/index.html#first-headline',
+          value: './index.html#first-headline',
+        },
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'https://example.com/fixtures/01-HtmlPage/index.html?data=in&query=params',
+          value: './index.html?data=in&query=params',
+        },
+      ],
+    });
+  });
+
+  it('03: images', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/03-images.html'),
+      withTestOptions({
+        localPath: fileURLToPath(new URL('fixtures/01-HtmlPage/03-images.html', import.meta.url)),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/03-images.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/03-images.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'src',
+          tag: 'img',
+          url: 'https://example.com/empty.png',
+          value: '/empty.png',
+        },
+        {
+          attribute: 'src',
+          tag: 'img',
+          url: 'https://example.com/fixtures/01-HtmlPage/empty.png',
+          value: './empty.png',
+        },
+        {
+          attribute: 'src',
+          tag: 'img',
+          url: 'https://example.com/fixtures/01-HtmlPage/empty.png?data=in&query=params',
+          value: './empty.png?data=in&query=params',
+        },
+      ],
+    });
+  });
+
+  it('04: picture with srcset', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/04-picture-srcset.html'),
+      withTestOptions({
+        localPath: fileURLToPath(
+          new URL('fixtures/01-HtmlPage/04-picture-srcset.html', import.meta.url),
+        ),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/04-picture-srcset.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/04-picture-srcset.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'src',
+          tag: 'img',
+          url: 'https://example.com/images/empty.png',
+          value: '/images/empty.png',
+        },
+        {
+          attribute: 'srcset',
+          tag: 'source',
+          url: 'https://example.com/images/empty-300.png',
+          value: '/images/empty-300.png 300w, /images/empty-600.png?data=in&query=params 600w',
+        },
+        {
+          attribute: 'srcset',
+          tag: 'source',
+          url: 'https://example.com/images/empty-600.png?data=in&query=params',
+          value: '/images/empty-300.png 300w, /images/empty-600.png?data=in&query=params 600w',
+        },
+        {
+          attribute: 'src',
+          tag: 'img',
+          url: 'https://example.com/images/missing.png',
+          value: '/images/missing.png',
+        },
+        {
+          attribute: 'srcset',
+          tag: 'source',
+          url: 'https://example.com/images/missing-300.png',
+          value: '/images/missing-300.png 300w, /images/missing-600.png 600w',
+        },
+        {
+          attribute: 'srcset',
+          tag: 'source',
+          url: 'https://example.com/images/missing-600.png',
+          value: '/images/missing-300.png 300w, /images/missing-600.png 600w',
+        },
+      ],
+    });
+  });
+
+  it('05: mailto', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/05-mailto.html'),
+      withTestOptions({
+        localPath: fileURLToPath(new URL('fixtures/01-HtmlPage/05-mailto.html', import.meta.url)),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/05-mailto.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/05-mailto.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'mailto:foo@bar.com',
+          value: 'mailto:foo@bar.com',
+        },
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'mailto:address@example.com',
+          value:
+            'mailto:address@example.com',
+        },
+      ],
+    });
+  });
+
+  it('06: not http schema', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/06-not-http-schema.html'),
+      withTestOptions({
+        localPath: fileURLToPath(
+          new URL('fixtures/01-HtmlPage/06-not-http-schema.html', import.meta.url),
+        ),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/06-not-http-schema.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/06-not-http-schema.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'sketch://add-library?url=https%3A%2F%2Fmyexample.com%2Fdesign%2Fui-kit.xml',
+          value: 'sketch://add-library?url=https%3A%2F%2Fmyexample.com%2Fdesign%2Fui-kit.xml',
+        },
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'vscode://file/c:/myProject/package.json:5:10',
+          value: 'vscode://file/c:/myProject/package.json:5:10',
+        },
+      ],
+    });
+  });
+
+  it('07: tel', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/07-tel.html'),
+      withTestOptions({
+        localPath: fileURLToPath(new URL('fixtures/01-HtmlPage/07-tel.html', import.meta.url)),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/07-tel.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/07-tel.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'tel:99999',
+          value: 'tel:99999',
+        },
+      ],
+    });
+  });
+
+  it('08: ignore about schema', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/08-ignore-about-schema.html'),
+      withTestOptions({
+        localPath: fileURLToPath(
+          new URL('fixtures/01-HtmlPage/08-ignore-about-schema.html', import.meta.url),
+        ),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/08-ignore-about-schema.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/08-ignore-about-schema.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'about:dino',
+          value: 'about:dino',
+        },
+      ],
+    });
+  });
+
+  it('09: html meta refresh', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/09-html-meta-refresh.html'),
+      withTestOptions({
+        localPath: fileURLToPath(
+          new URL('fixtures/01-HtmlPage/09-html-meta-refresh.html', import.meta.url),
+        ),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    // the HtmlPage starting it is not added as we only create it for the test
+    // but the redirect it found does get added to the asset manager
+    expect(page.options.assetManager?.size).to.equal(1);
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/09-html-meta-refresh.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/09-html-meta-refresh.html',
+        localSourcePath: '',
+      },
+      redirectTargetUrl: new URL('https://example.com/en/getting-started'),
+      hashes: [],
+      references: [],
+    });
+  });
+});

packages/check-website/test-node/02-gatherFiles.test.js

@@ -0,0 +1,68 @@
+import { expect } from 'chai';
+import path from 'path';
+import { gatherFiles } from '../src/helpers/gatherFiles.js';
+
+const currentDir = path.dirname(new URL(import.meta.url).pathname);
+
+/**
+ * @param {string[]} files
+ */
+function cleanupFiles(files) {
+  return files.map(file => (file ? `abs::${path.relative(currentDir, file)}` : file));
+}
+
+describe('gatherFiles', () => {
+  it('01: current dir', async () => {
+    const files = await gatherFiles(
+      new URL('fixtures/02-gatherFiles/01-current-dir/', import.meta.url),
+    );
+
+    expect(cleanupFiles(files)).to.deep.equal([
+      'abs::fixtures/02-gatherFiles/01-current-dir/a.html',
+      'abs::fixtures/02-gatherFiles/01-current-dir/b.js',
+      'abs::fixtures/02-gatherFiles/01-current-dir/c.html',
+    ]);
+  });
+
+  it('02: sub dir goes files first', async () => {
+    const files = await gatherFiles(
+      new URL('fixtures/02-gatherFiles/02-sub-dir/', import.meta.url),
+    );
+
+    expect(cleanupFiles(files)).to.deep.equal([
+      'abs::fixtures/02-gatherFiles/02-sub-dir/a.html',
+      'abs::fixtures/02-gatherFiles/02-sub-dir/z.html',
+      'abs::fixtures/02-gatherFiles/02-sub-dir/sub/b.html',
+      'abs::fixtures/02-gatherFiles/02-sub-dir/sub/c.html',
+      'abs::fixtures/02-gatherFiles/02-sub-dir/sub/some.js',
+      'abs::fixtures/02-gatherFiles/02-sub-dir/sub2/d.html',
+    ]);
+  });
+
+  it('03: index.html file always goes first', async () => {
+    const files = await gatherFiles(
+      new URL('fixtures/02-gatherFiles/03-index-first/', import.meta.url),
+    );
+
+    expect(cleanupFiles(files)).to.deep.equal([
+      'abs::fixtures/02-gatherFiles/03-index-first/index.html',
+      'abs::fixtures/02-gatherFiles/03-index-first/about.html',
+    ]);
+  });
+
+  it('03b: index.html file always goes first with sub dirs', async () => {
+    const files = await gatherFiles(
+      new URL('fixtures/02-gatherFiles/03b-index-first-sub-dir/', import.meta.url),
+    );
+
+    expect(cleanupFiles(files)).to.deep.equal([
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/index.html',
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/about.html',
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub/index.html',
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub/a.html',
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub2/index.html',
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub2/b.html',
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub2/z.html',
+    ]);
+  });
+});

packages/check-website/test-node/03-LocalReferencesPlugin.test.js

@@ -0,0 +1,105 @@
+import { expect } from 'chai';
+import { red, green } from 'colorette';
+
+import { LocalReferencesPlugin } from 'check-website';
+import { setupTestCli } from './test-helpers.js';
+
+function getOptions() {
+  return {
+    originUrl: 'https://example.com',
+    plugins: [new LocalReferencesPlugin()],
+  };
+}
+
+describe('LocalReferencesPlugin', () => {
+  it('01: finds a missing page', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/01-page-missing',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  it('01b: finds a variation of missing pages', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/01b-page-missing-variations',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('6 passed')}, ${red('4 failed')}`);
+  });
+
+  it('02: finds a missing hash', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/02-hash-missing',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  it('03: can identify full urls as internal', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/03-absolute-url-as-internal',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  it('04: automatically finds internal base url via canonical url', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/04-auto-finds-internal-base-url',
+      { plugins: [new LocalReferencesPlugin()] },
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('2 passed')}, ${red('1 failed')}`);
+  });
+
+  it('05: missing asset text file', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/05-asset-missing',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  it('06: starts crawling from the root index.html and ignores unlinked pages', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/06-crawls-from-root-index',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('2 failed')}`);
+  });
+
+  it('07: allow to mix urls with www and none www prefix', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/07-mix-www-and-none-www',
+      { plugins: [new LocalReferencesPlugin()] },
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('3 passed')}, ${red('1 failed')}`);
+  });
+
+  it('08: handle url in a case insensitive way', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/08-handle-urls-case-insensitive',
+      { plugins: [new LocalReferencesPlugin()] },
+      { captureLogs: true },
+    );
+    await execute();
+    // tests today-I-learned vs today-i-learned
+    expect(capturedLogs.join('\n')).to.include(`${green('2 passed')}, ${red('1 failed')}`);
+  });
+});

packages/check-website/test-node/04-ExternalReferencesPlugin.test.js

@@ -0,0 +1,49 @@
+import { expect } from 'chai';
+import { red, green } from 'colorette';
+
+import { ExternalReferencesPlugin } from 'check-website';
+import { setupTestCli } from './test-helpers.js';
+
+function getOptions() {
+  return {
+    originUrl: 'https://example.com',
+    plugins: [new ExternalReferencesPlugin()],
+  };
+}
+
+describe('ExternalReferencePlugin', () => {
+  it('01: finds a missing page', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/04-ExternalReferencePlugin/01-page-missing',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  // it('02: finds a missing hash', async () => {
+  //   const { cli } = await setupTestCli('fixtures/04-ExternalReferencePlugin/02-hash-missing');
+  //   await cli.start();
+  // });
+
+  it('03: image service using a none http url', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/04-ExternalReferencePlugin/03-image-service-none-http',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  it('04: data urls are not interpreted as external', async () => {
+    const { execute, getLastDynamicLog } = await setupTestCli(
+      'fixtures/04-ExternalReferencePlugin/04-data-urls-are-not-external',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(getLastDynamicLog()).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+});

packages/check-website/test-node/05-HasCanonicalPlugin.test.js

@@ -0,0 +1,34 @@
+import { expect } from 'chai';
+import { red, green } from 'colorette';
+
+import { HasCanonicalPlugin } from 'check-website';
+import { setupTestCli } from './test-helpers.js';
+
+function getOptions() {
+  return {
+    originUrl: 'https://example.com',
+    plugins: [new HasCanonicalPlugin()],
+  };
+}
+
+describe('HasCanonicalPlugin', () => {
+  it('01: with a canonical', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/05-HasCanonicalPlugin/01-with-canonical',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}`);
+  });
+
+  it('02: without a canonical', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/05-HasCanonicalPlugin/02-without-canonical',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${red('1 failed')}`);
+  });
+});

packages/check-website/test-node/06-config.test.js

@@ -0,0 +1,45 @@
+import { expect } from 'chai';
+import { red, green, gray } from 'colorette';
+
+import { LocalReferencesPlugin, ExternalReferencesPlugin } from 'check-website';
+import { setupTestCli } from './test-helpers.js';
+
+function getOptions() {
+  return {
+    originUrl: 'https://example.com',
+    plugins: [new LocalReferencesPlugin()],
+  };
+}
+
+describe('Config', () => {
+  it('01: reads config file', async () => {
+    const { execute, capturedLogs, cli } = await setupTestCli(
+      'fixtures/06-config/01-change-origin-url/site',
+      { plugins: [new LocalReferencesPlugin()] },
+      { captureLogs: true },
+    );
+    await execute();
+    expect(cli.options.originUrl).to.equal('https://some-domain.com');
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  it('02: supports skips', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/06-config/02-skips/site',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${gray('1 skipped')}`);
+  });
+
+  it('03: supports skips for external', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/06-config/03-skips-external/site',
+      { originUrl: 'https://example.com', plugins: [new ExternalReferencesPlugin()] },
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${gray('2 skipped')}`);
+  });
+});

packages/check-website/test-node/07-display.test.js

@@ -0,0 +1,32 @@
+import { expect } from 'chai';
+
+import { LocalReferencesPlugin, ExternalReferencesPlugin } from 'check-website';
+import { setupTestCli } from './test-helpers.js';
+
+function getOptions() {
+  return {
+    originUrl: 'https://example.com',
+    plugins: [new LocalReferencesPlugin(), new ExternalReferencesPlugin()],
+  };
+}
+
+describe('Display', () => {
+  it('01: duration is correct for multiple plugins', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/07-display/01-duration',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+
+    const lastDynamic = capturedLogs
+      .join('\n')
+      .split('────────────────────────────────────────────────────────────────────────────────')
+      // @ts-ignore
+      .at(-1);
+    const matches = lastDynamic.match(/\| 1\/1 links \| 🕑 (.*)s \|/g);
+
+    expect(matches).to.have.lengthOf(2);
+    expect(matches[0]).to.not.equal(matches[1]);
+  });
+});

packages/check-website/test-node/fixtures/01-AssetManager/page.html

@@ -0,0 +1 @@
+<a href="./file.txt">File</a>

packages/check-website/test-node/fixtures/01-HtmlPage/01-hashes.html

@@ -0,0 +1,5 @@
+<h1 id="first">First</h1>
+<p>Lorem ipsum</p>
+<h2 id="second">Second</h2>
+<input name="first-name" />
+<a href="#" name="third">Third</a>

packages/check-website/test-node/fixtures/01-HtmlPage/02-internal-link.html

@@ -0,0 +1,4 @@
+<a href="/index.html">index</a>
+<a href="./index.html">index</a>
+<a href="./index.html#first-headline">index</a>
+<a href="./index.html?data=in&query=params">index</a>

packages/check-website/test-node/fixtures/01-HtmlPage/03-images.html

@@ -0,0 +1,3 @@
+<img src="/empty.png" alt="" />
+<img src="./empty.png" alt="" />
+<img src="./empty.png?data=in&query=params" alt="" />

packages/check-website/test-node/fixtures/01-HtmlPage/04-picture-srcset.html

@@ -0,0 +1,9 @@
+<picture>
+  <source srcset="/images/empty-300.png 300w, /images/empty-600.png?data=in&query=params 600w" type="image/jpeg" sizes="(min-width: 62.5em) 25vw, (min-width: 30.625em) 50vw, 100vw">
+  <img src="/images/empty.png" alt="Empty" width="300" height="225">
+</picture>
+
+<picture>
+  <source srcset="/images/missing-300.png 300w, /images/missing-600.png 600w" type="image/jpeg" sizes="(min-width: 62.5em) 25vw, (min-width: 30.625em) 50vw, 100vw">
+  <img src="/images/missing.png" alt="Empty" width="300" height="225">
+</picture>

packages/check-website/test-node/fixtures/01-HtmlPage/05-mailto.html

@@ -0,0 +1,3 @@
+<a href="mailto:foo@bar.com"></a>
+<!-- encoded mailto links -->
+<a href="&#109;&#97;&#105;&#108;&#116;&#111;&#58;&#97;&#100;&#100;&#114;&#101;&#115;&#115;&#64;&#101;&#120;&#97;&#109;&#112;&#108;&#101;&#46;&#99;&#111;&#109;"></a>

packages/check-website/test-node/fixtures/01-HtmlPage/06-not-http-schema.html

@@ -0,0 +1,2 @@
+<a href="sketch://add-library?url=https%3A%2F%2Fmyexample.com%2Fdesign%2Fui-kit.xml"></a>
+<a href="vscode://file/c:/myProject/package.json:5:10"></a>

packages/check-website/test-node/fixtures/01-HtmlPage/07-tel.html

@@ -0,0 +1 @@
+<a href="tel:99999"></a>

packages/check-website/test-node/fixtures/01-HtmlPage/08-ignore-about-schema.html

@@ -0,0 +1 @@
+<a href="about:dino"></a>

packages/check-website/test-node/fixtures/01-HtmlPage/09-html-meta-refresh.html

@@ -0,0 +1,2 @@
+<!DOCTYPE html>
+<meta http-equiv="refresh" content="0;url=/en/getting-started" />