chore: fetch cache

Grammar edit from none to non-chromium browser(s)

.eslintignore

@@ -13,6 +13,10 @@ __output-dev
 
 docs/_merged*
 *-mdjs-generated.js
+*-converted-md-source.js
+*-converted-md.js
+
+__example_site-for-check-website
 
 # sanity example has a separate backend that is unrelated to Rocket
 # therefore it does not need to follow it code rules

.github/workflows/release.yml

@@ -12,45 +12,29 @@ jobs:
     name: Release
     runs-on: ubuntu-latest
     steps:
+      - uses: google/wireit@setup-github-actions-caching/v1
+
       - name: Checkout Repo
         uses: actions/checkout@master
         with:
           # This makes Actions fetch all Git history so that Changesets can generate changelogs with the correct commits
           fetch-depth: 0
 
-      - name: Setup Node.js 14.x
+      - name: Setup Node.js 18.x
         uses: actions/setup-node@master
         with:
-          node-version: 14.x
+          node-version: 18.x
           registry-url: 'https://registry.npmjs.org'
 
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-
-      - uses: actions/cache@v2
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
       - name: Install Dependencies
-        run: yarn --frozen-lockfile
-
-      - name: Build Packages
-        run: yarn build:packages
-
-      - name: Build Types
-        run: yarn types
+        run: npm ci
 
       - name: Create Release Pull Request or Publish to npm
         id: changesets
         uses: changesets/action@master
         with:
           # This expects you to have a script called release which does a build for your packages and calls changeset publish
-          publish: yarn release
+          publish: npm run release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/verify.yml

@@ -8,8 +8,9 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [18.x]
     steps:
+      - uses: google/wireit@setup-github-actions-caching/v1
       - uses: actions/checkout@v2
 
       - name: Setup Node ${{ matrix.node-version }}
@@ -17,20 +18,8 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-
-      - uses: actions/cache@v2
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
       - name: Install dependencies
-        run: yarn --frozen-lockfile
+        run: npm ci
 
       - name: Install Playwright dependencies
         run: npx playwright install-deps
@@ -38,40 +27,8 @@ jobs:
       - name: Install Playwright
         run: npx playwright install
 
-      - name: Build Packages
-        run: yarn build:packages
-
       - name: Lint
-        run: yarn lint
+        run: npm run lint
 
       - name: Test
-        run: yarn test
-
-  # verify-windows:
-  #   name: Verify windows
-  #   runs-on: windows-latest
-  #   steps:
-  #     - uses: actions/checkout@v2
-
-  #     - name: Setup Node 12.x
-  #       uses: actions/setup-node@v1
-  #       with:
-  #         node-version: 12.x
-
-  #     - name: Get yarn cache directory path
-  #       id: yarn-cache-dir-path
-  #       run: echo "::set-output name=dir::$(yarn cache dir)"
-
-  #     - uses: actions/cache@v2
-  #       id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-  #       with:
-  #         path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-  #         key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-  #         restore-keys: |
-  #           ${{ runner.os }}-yarn-
-
-  #     - name: Install dependencies
-  #       run: yarn --frozen-lockfile
-
-  #     - name: Test
-  #       run: yarn test
+        run: npm run test

.gitignore

@@ -12,21 +12,16 @@ coverage/
 ## npm
 node_modules
 npm-debug.log
-yarn-error.log
 
 ## temp folders
 /.tmp/
 
-## we prefer yarn.lock
-package-lock.json
-## lock files in packages we do not need to save
-packages/*/yarn.lock
-
 ## build output
 dist
 dist-types
 stats.html
 *.tsbuildinfo
+.wireit
 
 # Rocket Search
 rocket-search-index.json
@@ -49,3 +44,4 @@ docs_backup
 
 ## Local playground
 examples/testing
+__example_site-for-check-website

.vscode/settings.json

@@ -8,5 +8,5 @@
     "**/*-mdjs-generated.js": true,
     "**/dist-types": true,
   },
-  "editor.experimental.stickyScroll.enabled": true
+  "editor.stickyScroll.enabled": true
 }

CONTRIBUTING.md

@@ -32,10 +32,10 @@ git checkout -b my-awesome-fix
 
 ## Preparing Your Local Environment for Development
 
-Now that you have cloned the repository, ensure you have [yarn](https://classic.yarnpkg.com/lang/en/) installed, then run the following commands to set up the development environment.
+Now that you have cloned the repository, ensure you have [node](https://nodejs.org/) installed, then run the following commands to set up the development environment.
 
 ```shell
-yarn install
+npm install
 ```
 
 This will download and install all packages needed.
@@ -50,7 +50,7 @@ If you're making cross-package changes, you need to compile the TypeScript code.
 
 ### Running Tests
 
-To run the tests of a package, it's recommended to `cd` into the package directory and then using `yarn test` to run them. This way you're only running tests of that specific package.
+To run the tests of a package, it's recommended to `cd` into the package directory and then using `npm run test` to run them. This way you're only running tests of that specific package.
 
 ### Integration Testing
 
@@ -58,7 +58,7 @@ To see how your changes integrate with everything together you can use the `test
 
 ## Adding New Packages
 
-For all projects the tsconfig/jsconfig configuration files are auto generated. You need to add an entry to the [./workspace-packages.ts](./workspace-packages.ts) to let it generate a config for you. After adding an entry, run `yarn update-package-configs` to generate the files for you.
+For all projects the tsconfig/jsconfig configuration files are auto generated. You need to add an entry to the [./workspace-packages.ts](./workspace-packages.ts) to let it generate a config for you. After adding an entry, run `npm run update-package-configs` to generate the files for you.
 
 ## Creating a Changeset
 
@@ -70,7 +70,7 @@ This documents your intent to release, and allows you to specify a message that
 Run
 
 ```shell
-yarn changeset
+npm run changeset
 ```
 
 And use the menu to select for which packages you need a release, and then select what kind of release. For the release type, we follow [Semantic Versioning](https://semver.org/), so please take a look if you're unfamiliar.

README.md

@@ -28,7 +28,7 @@
 <p align="center">
   <a href="https://rocket.modern-web.dev">Website</a>
   ·
-  <a href="https://rocket.modern-web.dev/doc/">Documentation</a>
+  <a href="https://rocket.modern-web.dev/docs/">Documentation</a>
   ·
   <a href="https://rocket.modern-web.dev/chat">Discord Community</a>
 </p>

examples/01-hydration-starter/.gitignore

@@ -3,8 +3,6 @@ node_modules/
 
 # logs
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 
 # environment variables
 .env

examples/01-hydration-starter/package.json

@@ -7,7 +7,7 @@
   "scripts": {
     "build": "rocket build",
     "dev": "npm start",
-    "preview": "rocket preview",
+    "preview": "rocket preview --open",
     "start": "NODE_DEBUG=engine:rendering rocket start --open"
   },
   "devDependencies": {

examples/02-blog-starter/.gitignore

@@ -3,8 +3,6 @@ node_modules/
 
 # logs
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 
 # environment variables
 .env

examples/02-blog-starter/package.json

@@ -12,7 +12,7 @@
   "scripts": {
     "build": "rocket build",
     "dev": "npm start",
-    "preview": "rocket preview",
+    "preview": "rocket preview --open",
     "start": "NODE_DEBUG=engine:rendering rocket start --open"
   },
   "devDependencies": {

examples/02-blog-starter/site/pages/pageTreeData.rocketGenerated.json

@@ -1,8 +1,8 @@
 {
   "title": "Example Blog",
-  "h1": "\n            \n          ",
+  "h1": "My Blog",
   "name": "Example Blog",
-  "menuLinkText": "\n            \n          ",
+  "menuLinkText": "My Blog",
   "url": "/",
   "outputRelativeFilePath": "index.html",
   "sourceRelativeFilePath": "index.rocket.js",
@@ -20,7 +20,7 @@
   "children": [
     {
       "title": "",
-      "h1": "\n            \n          ",
+      "h1": "My Blog",
       "headlinesWithId": [
         {
           "text": "About",
@@ -28,8 +28,8 @@
           "level": 1
         }
       ],
-      "name": "\n            \n          ",
-      "menuLinkText": "\n            \n          ",
+      "name": "My Blog",
+      "menuLinkText": "My Blog",
       "url": "/about/",
       "outputRelativeFilePath": "about/index.html",
       "sourceRelativeFilePath": "about.rocket.md",
@@ -60,9 +60,9 @@
       "children": [
         {
           "title": "Hello world!",
-          "h1": "\n            \n          ",
+          "h1": "My Blog",
           "name": "Hello world!",
-          "menuLinkText": "\n            \n          ",
+          "menuLinkText": "My Blog",
           "url": "/blog/hello-world/",
           "outputRelativeFilePath": "blog/hello-world/index.html",
           "sourceRelativeFilePath": "blog/hello-world.rocket.md",
@@ -82,9 +82,9 @@
         },
         {
           "title": "With Image!",
-          "h1": "\n            \n          ",
+          "h1": "My Blog",
           "name": "With Image!",
-          "menuLinkText": "\n            \n          ",
+          "menuLinkText": "My Blog",
           "url": "/blog/with-image/",
           "outputRelativeFilePath": "blog/with-image/index.html",
           "sourceRelativeFilePath": "blog/with-image.rocket.md",

examples/03-minimal-starter/.gitignore

@@ -3,8 +3,6 @@ node_modules/
 
 # logs
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 
 # environment variables
 .env

examples/03-minimal-starter/package.json

@@ -7,7 +7,7 @@
   "scripts": {
     "build": "rocket build",
     "dev": "npm start",
-    "preview": "rocket preview",
+    "preview": "rocket preview --open",
     "start": "NODE_DEBUG=engine:rendering rocket start --open"
   },
   "devDependencies": {

examples/04-sanity-minimal-starter/backend/.gitignore

@@ -1,4 +1,2 @@
 /dist
 /node_modules
-
-yarn.lock

examples/04-sanity-minimal-starter/backend/package.json

@@ -3,7 +3,6 @@
   "private": true,
   "version": "1.0.0",
   "description": "",
-  "main": "package.json",
   "author": "Jaydan Urwin <jaydan@jaydanurwin.com>",
   "license": "UNLICENSED",
   "scripts": {

examples/04-sanity-minimal-starter/frontend/.gitignore

@@ -3,8 +3,6 @@ node_modules/
 
 # logs
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 
 # environment variables
 .env

examples/04-sanity-minimal-starter/frontend/package.json

@@ -7,7 +7,7 @@
   "scripts": {
     "build": "rocket build",
     "dev": "rocket start --open",
-    "preview": "rocket preview",
+    "preview": "rocket preview --open",
     "start": "NODE_DEBUG=engine:rendering rocket start --open"
   },
   "devDependencies": {

examples/50-landing-theme-spark/.gitignore

@@ -3,8 +3,6 @@ node_modules/
 
 # logs
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 
 # environment variables
 .env

examples/50-landing-theme-spark/package.json

@@ -7,7 +7,7 @@
   "scripts": {
     "build": "rocket build",
     "dev": "npm start",
-    "preview": "rocket preview",
+    "preview": "rocket preview --open",
     "start": "NODE_DEBUG=engine:rendering rocket start --open"
   },
   "devDependencies": {

examples/50-landing-theme-spark/site/pages/aliens.rocket.md

@@ -0,0 +1,24 @@
+```js server
+/* START - Rocket auto generated - do not touch */
+export const sourceRelativeFilePath = 'aliens.rocket.md';
+import { layout, components } from './recursive.data.js';
+export { layout, components };
+export async function registerCustomElements() {
+  // server-only components
+  // prettier-ignore
+  customElements.define('rocket-content-area', await import('@rocket/components/content-area.js').then(m => m.RocketContentArea));
+  // prettier-ignore
+  customElements.define('rocket-header-scroll-menu', await import('@rocket/components/header-scroll-menu.js').then(m => m.RocketHeaderScrollMenu));
+  // prettier-ignore
+  customElements.define('rocket-columns', await import('@rocket/components/columns.js').then(m => m.RocketColumns));
+}
+/* END - Rocket auto generated - do not touch */
+
+export const title = 'Aliens';
+```
+
+Vastness is bearable only through love Cambrian explosion a still more glorious dawn awaits Euclid consciousness extraordinary claims require extraordinary evidence. Realm of the galaxies invent the universe culture made in the interiors of collapsing stars Drake Equation with pretty stories for which there's little good evidence. Bits of moving fluff preserve and cherish that pale blue dot shores of the cosmic ocean the ash of stellar alchemy brain is the seed of intelligence courage of our questions.
+
+With pretty stories for which there's little good evidence not a sunrise but a galaxyrise rich in heavy atoms consciousness the sky calls to us rings of Uranus. Shores of the cosmic ocean shores of the cosmic ocean a very small stage in a vast cosmic arena of brilliant syntheses laws of physics muse about? Invent the universe finite but unbounded extraordinary claims require extraordinary evidence made in the interiors of collapsing stars muse about invent the universe.
+
+The ash of stellar alchemy Cambrian explosion how far away kindling the energy hidden in matter cosmic ocean descended from astronomers. The carbon in our apple pies a very small stage in a vast cosmic arena brain is the seed of intelligence the only home we've ever known the carbon in our apple pies the carbon in our apple pies? Corpus callosum muse about citizens of distant epochs finite but unbounded extraordinary claims require extraordinary evidence finite but unbounded and billions upon billions upon billions upon billions upon billions upon billions upon billions.

examples/50-landing-theme-spark/site/pages/humans.rocket.md

@@ -0,0 +1,30 @@
+```js server
+/* START - Rocket auto generated - do not touch */
+export const sourceRelativeFilePath = 'humans.rocket.md';
+import { layout, components } from './recursive.data.js';
+export { layout, components };
+export async function registerCustomElements() {
+  // server-only components
+  // prettier-ignore
+  customElements.define('rocket-content-area', await import('@rocket/components/content-area.js').then(m => m.RocketContentArea));
+  // prettier-ignore
+  customElements.define('rocket-header-scroll-menu', await import('@rocket/components/header-scroll-menu.js').then(m => m.RocketHeaderScrollMenu));
+  // prettier-ignore
+  customElements.define('rocket-columns', await import('@rocket/components/columns.js').then(m => m.RocketColumns));
+}
+/* END - Rocket auto generated - do not touch */
+
+export const title = 'Humans';
+```
+
+Astonishment dispassionate extraterrestrial observer Drake Equation radio telescope Hypatia of brilliant syntheses. Vastness is bearable only through love Vangelis a mote of dust suspended in a sunbeam rings of Uranus vanquish the impossible rings of Uranus? Emerged into consciousness a billion trillion laws of physics emerged into consciousness bits of moving fluff as a patch of light. Sea of Tranquility something incredible is waiting to be known permanence of the stars something incredible is waiting to be known network of wormholes invent the universe.
+
+Orion's sword the carbon in our apple pies tesseract made in the interiors of collapsing stars take root and flourish Jean-François Champollion. The ash of stellar alchemy rich in heavy atoms tingling of the spine invent the universe shores of the cosmic ocean star stuff harvesting star light. Bits of moving fluff courage of our questions as a patch of light preserve and cherish that pale blue dot citizens of distant epochs concept of the number one.
+
+Two ghostly white figures in coveralls and helmets are softly dancing dispassionate extraterrestrial observer tesseract finite but unbounded the sky calls to us shores of the cosmic ocean? Muse about tingling of the spine stirred by starlight muse about not a sunrise but a galaxyrise venture. Concept of the number one are creatures of the cosmos something incredible is waiting to be known courage of our questions the only home we've ever known rich in heavy atoms.
+
+Across the centuries light years billions upon billions bits of moving fluff permanence of the stars consciousness. Shores of the cosmic ocean realm of the galaxies vastness is bearable only through love Sea of Tranquility network of wormholes are creatures of the cosmos? Extraordinary claims require extraordinary evidence finite but unbounded made in the interiors of collapsing stars network of wormholes finite but unbounded radio telescope. Invent the universe from which we spring with pretty stories for which there's little good evidence Euclid inconspicuous motes of rock and gas something incredible is waiting to be known.
+
+Birth dream of the mind's eye prime number at the edge of forever a billion trillion permanence of the stars? Vanquish the impossible two ghostly white figures in coveralls and helmets are softly dancing vastness is bearable only through love citizens of distant epochs something incredible is waiting to be known hydrogen atoms. Two ghostly white figures in coveralls and helmets are softly dancing shores of the cosmic ocean the ash of stellar alchemy Apollonius of Perga vastness is bearable only through love network of wormholes.
+
+Gathered by gravity Sea of Tranquility galaxies astonishment hearts of the stars venture. Citizens of distant epochs the carbon in our apple pies encyclopaedia galactica birth brain is the seed of intelligence permanence of the stars? A still more glorious dawn awaits concept of the number one a mote of dust suspended in a sunbeam dream of the mind's eye Euclid a very small stage in a vast cosmic arena and billions upon billions upon billions upon billions upon billions upon billions upon billions.

examples/50-landing-theme-spark/site/pages/index.rocket.html

@@ -128,30 +128,32 @@ export const needsLoader = true;
 
   <rocket-columns>
     <rocket-card>
-      <h4 slot="title">No astrophysicist</h4>
+      <h4 slot="title">Life</h4>
       <p>
-        would deny the possibility of life. I think we're not creative enough to imagine what life
-        would be like on another planet. Show me a dead alien. Better yet, show me a live one!
+        As a scientist, I want to go to Mars and back to asteroids and the Moon because I'm a
+        scientist. But I can tell you, I'm not so naive a scientist to think that the nation might
+        not have geopolitical reasons for going into space.
       </p>
-      <a slot="cta" class="cta" href="./life.rocket.html">Details</a>
+      <a slot="cta" class="cta" href="./life.rocket.md">Details</a>
     </rocket-card>
 
     <rocket-card>
-      <h4 slot="title">No astrophysicist</h4>
+      <h4 slot="title">Aliens</h4>
       <p>
-        would deny the possibility of life. I think we're not creative enough to imagine what life
-        would be like on another planet. Show me a dead alien. Better yet, show me a live one!
+        Vastness is bearable only through love Cambrian explosion a still more glorious dawn awaits
+        Euclid consciousness extraordinary claims require extraordinary evidence.
       </p>
-      <a slot="cta" class="cta" href="./life.rocket.html">Details</a>
+      <a slot="cta" class="cta" href="./aliens.rocket.md">Details</a>
     </rocket-card>
 
     <rocket-card>
-      <h4 slot="title">No astrophysicist</h4>
+      <h4 slot="title">Humans</h4>
       <p>
-        would deny the possibility of life. I think we're not creative enough to imagine what life
-        would be like on another planet. Show me a dead alien. Better yet, show me a live one!
+        Astonishment dispassionate extraterrestrial observer Drake Equation radio telescope Hypatia
+        of brilliant syntheses. Vastness is bearable only through love Vangelis a mote of dust
+        suspended in a sunbeam rings of Uranus vanquish the impossible rings of Uranus?
       </p>
-      <a slot="cta" class="cta" href="./life.rocket.html">Details</a>
+      <a slot="cta" class="cta" href="./humans.rocket.md">Details</a>
     </rocket-card>
   </rocket-columns>
 </rocket-content-area>

examples/50-landing-theme-spark/site/pages/pageTreeData.rocketGenerated.json

@@ -1,24 +1,24 @@
 {
   "title": "Spark Rocket Example Template",
-  "h1": "\n        \n        \n      ",
+  "h1": "Home",
   "headlinesWithId": [
     {
       "text": "Home",
       "id": "home",
       "level": 1,
-      "rawText": "\n        \n        \n      "
+      "rawText": "@keyframes textAnimIn {\n        0% {\n          transform: translate3d(0, -120%, 0);\n        }\n\n        100% {\n          transform: translate3d(0, 0%, 0);\n        }\n      }\n\n      @keyframes textAnimOut {\n        0% {\n          transform: translate3d(0, 0%, 0);\n        }\n\n        50% {\n          transform: translate3d(0, -20%, 0);\n        }\n\n        100% {\n          transform: translate3d(0, 120%, 0);\n        }\n      }\n\n      p {\n        position: absolute;\n        top: 0;\n        margin: 0;\n        transform: translate3d(0, -120%, 0);\n      }\n\n      .anim-static {\n        transform: translate3d(0, 0, 0);\n      }\n\n      .anim-in {\n        animation: textAnimIn 0.6s 0.3s forwards;\n      }\n\n      .anim-out {\n        animation: textAnimOut 0.6s forwards;\n      }\n\n      :host {\n        position: block;\n        position: relative;\n        overflow: hidden;\n        display: block;\n        height: 1.5em;\n      }\n      div {\n        display: flex;\n        max-width: 960px;\n        margin: 0 auto;\n      }\n      slot {\n        display: block;\n        flex: 1;\n      }\n    \n          Welcome\n        \n          Hello\n        \n          Aloa\n        lit-part-->\n        to the world of science."
     },
     {
       "text": "Credit",
       "id": "credit",
       "level": 2,
-      "rawText": "\n    People credit me for \n    \n  "
+      "rawText": "People credit me for \n    making the universe interesting,"
     },
     {
       "text": "Knowledge",
       "id": "knowledge",
       "level": 2,
-      "rawText": "\n    I've known from long ago \n    \n  "
+      "rawText": "I've known from long ago \n    that the universe was calling me"
     },
     {
       "text": "Sign Up",
@@ -30,7 +30,7 @@
       "text": "Say",
       "id": "say",
       "level": 2,
-      "rawText": "\n    What others  \n    about us?\n  "
+      "rawText": "What others say \n    about us?"
     },
     {
       "text": "FAQ",
@@ -39,7 +39,7 @@
     }
   ],
   "name": "Spark Rocket Example Template",
-  "menuLinkText": "\n        \n        \n      ",
+  "menuLinkText": "Home",
   "url": "/",
   "outputRelativeFilePath": "index.html",
   "sourceRelativeFilePath": "index.rocket.html",
@@ -69,6 +69,70 @@
   "description": "Everyone can code a website!",
   "needsLoader": true,
   "children": [
+    {
+      "title": "Aliens",
+      "h1": "Aliens",
+      "name": "Aliens",
+      "menuLinkText": "Aliens",
+      "url": "/aliens/",
+      "outputRelativeFilePath": "aliens/index.html",
+      "sourceRelativeFilePath": "aliens.rocket.md",
+      "level": 1,
+      "components": {
+        "rocket-columns": "@rocket/components/columns.js::RocketColumns",
+        "rocket-card": "@rocket/components/card.js::RocketCard",
+        "rocket-content-area": "@rocket/components/content-area.js::RocketContentArea",
+        "rocket-details": "@rocket/components/details.js::RocketDetails",
+        "rocket-drawer": "@rocket/components/drawer.js::RocketDrawer",
+        "rocket-feature-small": "@rocket/components/feature-small.js::RocketFeatureSmall",
+        "rocket-header": "@rocket/components/header.js::RocketHeader",
+        "rocket-header-scroll-menu": "@rocket/components/header-scroll-menu.js::RocketHeaderScrollMenu",
+        "rocket-icon": "@rocket/components/icon.js::RocketIcon",
+        "rocket-icon-card": "@rocket/components/icon-card.js::RocketIconCard",
+        "rocket-main": "@rocket/components/main.js::RocketMain",
+        "rocket-main-docs": "@rocket/components/main-docs.js::RocketMainDocs",
+        "rocket-opengraph-overview": "@rocket/components/open-graph-overview.js::RocketOpenGraphOverview",
+        "rocket-rotating-text": "@rocket/components/rotating-text.js::RocketRotatingText",
+        "rocket-social-link": "@rocket/components/social-link.js::RocketSocialLink",
+        "rocket-testimonial-small": "@rocket/components/testimonial-small.js::RocketTestimonialSmall",
+        "inline-notification": "@rocket/components/inline-notification.js::InlineNotification",
+        "permanent-notification": "@rocket/components/permanent-notification.js::PermanentNotification",
+        "block-blue": "@rocket/spark/block-blue.js::BlockBlue",
+        "block-features": "@rocket/spark/block-features.js::BlockFeatures"
+      }
+    },
+    {
+      "title": "Humans",
+      "h1": "Humans",
+      "name": "Humans",
+      "menuLinkText": "Humans",
+      "url": "/humans/",
+      "outputRelativeFilePath": "humans/index.html",
+      "sourceRelativeFilePath": "humans.rocket.md",
+      "level": 1,
+      "components": {
+        "rocket-columns": "@rocket/components/columns.js::RocketColumns",
+        "rocket-card": "@rocket/components/card.js::RocketCard",
+        "rocket-content-area": "@rocket/components/content-area.js::RocketContentArea",
+        "rocket-details": "@rocket/components/details.js::RocketDetails",
+        "rocket-drawer": "@rocket/components/drawer.js::RocketDrawer",
+        "rocket-feature-small": "@rocket/components/feature-small.js::RocketFeatureSmall",
+        "rocket-header": "@rocket/components/header.js::RocketHeader",
+        "rocket-header-scroll-menu": "@rocket/components/header-scroll-menu.js::RocketHeaderScrollMenu",
+        "rocket-icon": "@rocket/components/icon.js::RocketIcon",
+        "rocket-icon-card": "@rocket/components/icon-card.js::RocketIconCard",
+        "rocket-main": "@rocket/components/main.js::RocketMain",
+        "rocket-main-docs": "@rocket/components/main-docs.js::RocketMainDocs",
+        "rocket-opengraph-overview": "@rocket/components/open-graph-overview.js::RocketOpenGraphOverview",
+        "rocket-rotating-text": "@rocket/components/rotating-text.js::RocketRotatingText",
+        "rocket-social-link": "@rocket/components/social-link.js::RocketSocialLink",
+        "rocket-testimonial-small": "@rocket/components/testimonial-small.js::RocketTestimonialSmall",
+        "inline-notification": "@rocket/components/inline-notification.js::InlineNotification",
+        "permanent-notification": "@rocket/components/permanent-notification.js::PermanentNotification",
+        "block-blue": "@rocket/spark/block-blue.js::BlockBlue",
+        "block-features": "@rocket/spark/block-features.js::BlockFeatures"
+      }
+    },
     {
       "title": "Life",
       "h1": "Life",

examples/50-landing-theme-spark/site/src/layouts/layoutData.js

@@ -21,7 +21,7 @@ export const layoutData = {
           Modern Web<br />
           Internet 12<br />
           0000 Web<br />
-          <a href="#">office@modern-web.dev</a><br />
+          <a href="#">hello@modern-web.dev</a><br />
           <a href="#">0000 / 11223344</a>
         </div>
 

examples/51-docs-theme-launch/.gitignore

@@ -3,8 +3,6 @@ node_modules/
 
 # logs
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
 
 # environment variables
 .env

examples/51-docs-theme-launch/package.json

@@ -7,7 +7,7 @@
   "scripts": {
     "build": "rocket build",
     "dev": "npm start",
-    "preview": "rocket preview",
+    "preview": "rocket preview --open",
     "start": "NODE_DEBUG=engine:rendering rocket start --open"
   },
   "devDependencies": {

examples/51-docs-theme-launch/site/pages/40--blog/index.rocket.js

@@ -12,6 +12,10 @@ export async function registerCustomElements() {
     'rocket-header',
     await import('@rocket/components/header.js').then(m => m.RocketHeader),
   );
+  customElements.define(
+    'launch-blog-preview',
+    await import('@rocket/launch/blog-preview.js').then(m => m.LaunchBlogPreview),
+  );
   customElements.define(
     'launch-blog-overview',
     await import('@rocket/launch/blog-overview.js').then(m => m.LaunchBlogOverview),

examples/51-docs-theme-launch/site/pages/pageTreeData.rocketGenerated.json

@@ -1,8 +1,8 @@
 {
   "title": "Welcome to Rocket",
-  "h1": "\n        \n      ",
-  "name": "\n        \n      ",
-  "menuLinkText": "\n        \n      ",
+  "h1": "",
+  "name": "Welcome to Rocket",
+  "menuLinkText": "Welcome to Rocket",
   "url": "/",
   "outputRelativeFilePath": "index.html",
   "sourceRelativeFilePath": "index.rocket.js",

package.json

@@ -10,8 +10,7 @@
   "scripts": {
     "analyze": "run-s analyze:* format:*",
     "analyze:analyze": "node scripts/workspaces-scripts-bin.mjs analyze",
-    "build": "npm run build:packages && npm run rocket:build",
-    "build:packages": "node scripts/workspaces-scripts-bin.mjs build:package",
+    "build": "npm run rocket:build",
     "build:site": "run-s analyze:* rocket:build",
     "changeset": "changeset",
     "debug": "web-test-runner --watch --config web-test-runner-chrome.config.mjs",
@@ -25,36 +24,30 @@
     "lint:prettier": "node node_modules/prettier/bin-prettier.js \"**/*.{ts,js,mjs,cjs,md}\" --check --ignore-path .eslintignore",
     "lint:types": "npm run types",
     "lint:versions": "node scripts/lint-versions.js",
-    "postinstall": "npm run setup",
-    "release": "changeset publish && yarn format",
+    "postinstall": "npx patch-package",
+    "preview": "node packages/cli/src/cli.js preview --open",
+    "release": "changeset publish && npm run format",
     "rocket:build": "NODE_DEBUG=engine:rendering node --trace-warnings packages/cli/src/cli.js build",
     "rocket:upgrade": "node packages/cli/src/cli.js upgrade",
     "search": "node packages/cli/src/cli.js search",
-    "setup": "npm run setup:ts-configs",
-    "setup:patches": "npx patch-package",
-    "setup:ts-configs": "node scripts/generate-ts-configs.mjs",
-    "start:experimental": "NODE_DEBUG=engine:rendering node --no-warnings --experimental-loader ./packages/engine/src/litCssLoader.js packages/cli/src/cli.js start --open",
     "start": "NODE_DEBUG=engine:rendering node --trace-warnings packages/cli/src/cli.js start --open",
-    "test": "yarn test:node && yarn test:web",
-    "test:integration": "playwright test packages/*/test-node/*.spec.js",
-    "test:node": "yarn test:unit && yarn test:integration",
+    "start:experimental": "NODE_DEBUG=engine:rendering node --no-warnings --experimental-loader ./packages/engine/src/litCssLoader.js packages/cli/src/cli.js start --open",
+    "test": "npm run test:node && npm run test:web",
+    "test:integration": "playwright test packages/*/test-node/*.spec.js --retries=3",
+    "test:node": "npm run test:unit && npm run test:integration",
     "test:unit": "node --trace-warnings ./node_modules/.bin/mocha --require ./scripts/testMochaGlobalHooks.js \"packages/*/test-node/**/*.test.{ts,js,mjs,cjs}\" -- --timeout 8000 --reporter dot --exit",
     "test:web": "web-test-runner",
-    "types": "run-s types:clear types:copy types:build",
-    "types:build": "tsc --build",
-    "types:clear": "rimraf packages/*/dist-types/",
-    "types:copy": "node scripts/workspaces-scripts-bin.mjs types:copy",
+    "types": "npm run types --workspaces --if-present",
     "update-dependency": "node scripts/update-dependency.js",
-    "update-esm-entrypoints": "node scripts/update-esm-entrypoints.mjs && yarn format",
-    "update-package-configs": "node scripts/update-package-configs.mjs && yarn format",
-    "xprestart": "yarn analyze"
+    "update-esm-entrypoints": "node scripts/update-esm-entrypoints.mjs && npm run format",
+    "update-package-configs": "node scripts/update-package-configs.mjs && npm run format",
+    "xprestart": "npm run analyze"
   },
-  "dependencies": {},
   "devDependencies": {
     "@changesets/cli": "^2.20.0",
     "@custom-elements-manifest/analyzer": "^0.4.12",
-    "@playwright/test": "^1.18.1",
     "@open-wc/testing": "^3.1.2",
+    "@playwright/test": "^1.18.1",
     "@rollup/plugin-commonjs": "^17.0.0",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-typescript": "^8.1.0",
@@ -80,7 +73,7 @@
     "husky": "^4.3.7",
     "lint-staged": "^10.5.3",
     "mocha": "^9.1.3",
-    "node-fetch": "^2.6.7",
+    "node-fetch": "^3.0.0",
     "npm-run-all": "^4.1.5",
     "onchange": "^7.1.0",
     "prettier": "^2.5.1",
@@ -93,7 +86,8 @@
     "rollup-plugin-terser": "^7.0.2",
     "sinon": "^9.2.3",
     "ts-node": "^9.1.1",
-    "typescript": "^4.6.3"
+    "typescript": "^4.8.4",
+    "wireit": "^0.7.2"
   },
   "eslintConfig": {
     "parser": "@typescript-eslint/parser",
@@ -117,7 +111,8 @@
       "@typescript-eslint/no-explicit-any": "off",
       "@typescript-eslint/no-empty-interface": "off",
       "@typescript-eslint/no-unused-vars": "error",
-      "@typescript-eslint/no-var-requires": "off"
+      "@typescript-eslint/no-var-requires": "off",
+      "@typescript-eslint/ban-ts-comment": "off"
     }
   },
   "husky": {

packages/building-rollup/index.js

@@ -1,12 +0,0 @@
-// /**
-//  * @typedef {import('./src/types').BasicOptions} BasicOptions
-//  * @typedef {import('./src/types').SpaOptions} SpaOptions
-//  */
-
-export { createBasicConfig, createBasicMetaConfig } from './src/createBasicConfig.js';
-export { createSpaConfig, createSpaMetaConfig } from './src/createSpaConfig.js';
-export { createMpaConfig, createMpaMetaConfig } from './src/createMpaConfig.js';
-export {
-  createServiceWorkerConfig,
-  createServiceWorkerMetaConfig,
-} from './src/createServiceWorkerConfig.js';

packages/building-rollup/package.json

@@ -13,10 +13,12 @@
   },
   "author": "Modern Web <hello@modern-web.dev> (https://modern-web.dev/)",
   "homepage": "https://rocket.modern-web.dev/docs/tools/building-rollup/",
-  "main": "./index.js",
   "type": "module",
   "exports": {
-    ".": "./index.js"
+    ".": {
+      "types": "./dist-types/src/index.d.ts",
+      "default": "./src/index.js"
+    }
   },
   "scripts": {
     "build:babelrc": "rimraf dist && rollup -c demo/babelrc/rollup.config.js",
@@ -27,6 +29,7 @@
     "build:spa-js-input": "rimraf dist && rollup -c demo/js/rollup.spa-js-input.config.js",
     "build:spa-nomodule": "rimraf dist && rollup -c demo/js/rollup.spa-nomodule.config.js",
     "build:ts": "rimraf dist && rollup -c demo/ts/rollup.spa.config.js",
+    "prepublishOnly": "npm run types",
     "start:babelrc": "npm run build:babelrc && npm run start:build",
     "start:build": "web-dev-server --root-dir dist --compatibility none --open",
     "start:cjs": "npm run build:cjs && npm run start:build",
@@ -38,10 +41,11 @@
     "start:ts": "npm run build:ts && npm run start:build",
     "start:watch": "npm run build:spa-nomodule -- --watch & npm run start:build",
     "test": "npm run test:node",
-    "test:node": "mocha test-node/**/*.test.js --timeout 5000"
+    "test:node": "mocha test-node/**/*.test.js --timeout 5000",
+    "types": "wireit"
   },
   "files": [
-    "*.js",
+    "dist-types",
     "src"
   ],
   "keywords": [
@@ -54,17 +58,35 @@
     "@babel/core": "^7.12.10",
     "@babel/preset-env": "^7.12.11",
     "@rollup/plugin-babel": "^5.2.2",
-    "@rollup/plugin-node-resolve": "^11.0.1",
-    "@rollup/plugin-replace": "^2.4.2",
+    "@rollup/plugin-node-resolve": "^15.0.1",
+    "@rollup/plugin-replace": "^5.0.1",
     "@web/rollup-plugin-html": "^1.8.0",
     "@web/rollup-plugin-import-meta-assets": "^1.0.4",
     "@web/rollup-plugin-polyfills-loader": "^1.1.0",
     "browserslist": "^4.16.1",
+    "plugins-manager": "^0.3.1",
     "rollup-plugin-terser": "^7.0.2",
     "workbox-broadcast-update": "^6.1.5",
     "workbox-cacheable-response": "^6.1.5",
     "workbox-expiration": "^6.1.5",
     "workbox-routing": "^6.1.5",
     "workbox-strategies": "^6.1.5"
+  },
+  "wireit": {
+    "types": {
+      "command": "copyfiles \"./types/**/*.d.ts\" dist-types/ && tsc --build --pretty",
+      "dependencies": [
+        "../plugins-manager:types"
+      ],
+      "clean": "if-file-deleted",
+      "files": [
+        "src/**/*.js",
+        "tsconfig.json"
+      ],
+      "output": [
+        "dist-types/**",
+        ".tsbuildinfo"
+      ]
+    }
   }
 }

packages/building-rollup/src/createBasicConfig.js

@@ -1,4 +1,5 @@
 import resolve from '@rollup/plugin-node-resolve';
+// @ts-ignore
 import { terser } from 'rollup-plugin-terser';
 import babelPkg from '@rollup/plugin-babel';
 
@@ -6,11 +7,19 @@ import { applyPlugins } from 'plugins-manager';
 
 const { babel } = babelPkg;
 
+/** @typedef {import('../types/main.js').BuildingRollupOptions} BuildingRollupOptions */
+
+/**
+ * @param {BuildingRollupOptions} [userConfig]
+ */
 export function createBasicConfig(userConfig) {
   const { config, metaPlugins } = createBasicMetaConfig(userConfig);
   return applyPlugins(config, metaPlugins);
 }
 
+/**
+ * @param {BuildingRollupOptions} [userConfig]
+ */
 export function createBasicMetaConfig(userConfig = { output: {} }) {
   const developmentMode =
     typeof userConfig.developmentMode !== 'undefined'
@@ -37,8 +46,12 @@ export function createBasicMetaConfig(userConfig = { output: {} }) {
     },
   };
 
+  /**
+   * @type {import('plugins-manager').MetaPlugin<any>[]}
+   */
   let metaPlugins = [
     {
+      // @ts-ignore
       plugin: resolve,
       options: {
         moduleDirectories: ['node_modules', 'web_modules'],
@@ -72,6 +85,7 @@ export function createBasicMetaConfig(userConfig = { output: {} }) {
     },
     {
       plugin: terser,
+      options: {},
     },
   ];
 

packages/building-rollup/src/createMpaConfig.js

@@ -1,7 +1,13 @@
 import { createSpaMetaConfig } from './createSpaConfig.js';
 import { adjustPluginOptions, applyPlugins } from 'plugins-manager';
+// @ts-ignore
 import { rollupPluginHTML } from '@web/rollup-plugin-html';
 
+/** @typedef {import('../types/main.js').BuildingRollupOptions} BuildingRollupOptions */
+
+/**
+ * @param {BuildingRollupOptions} [userConfig]
+ */
 export function createMpaConfig(userConfig) {
   const { config, metaPlugins } = createMpaMetaConfig(userConfig);
 
@@ -9,6 +15,9 @@ export function createMpaConfig(userConfig) {
   return final;
 }
 
+/**
+ * @param {BuildingRollupOptions} userConfig
+ */
 export function createMpaMetaConfig(userConfig = { output: {}, setupPlugins: [] }) {
   const { config, metaPlugins } = createSpaMetaConfig(userConfig);
 

packages/building-rollup/src/createServiceWorkerConfig.js

@@ -1,4 +1,5 @@
 import resolve from '@rollup/plugin-node-resolve';
+// @ts-ignore
 import { terser } from 'rollup-plugin-terser';
 import babelPkg from '@rollup/plugin-babel';
 import replace from '@rollup/plugin-replace';
@@ -7,11 +8,19 @@ import { applyPlugins } from 'plugins-manager';
 
 const { babel } = babelPkg;
 
+/** @typedef {import('../types/main.js').BuildingRollupOptions} BuildingRollupOptions */
+
+/**
+ * @param {BuildingRollupOptions} userConfig
+ */
 export function createServiceWorkerConfig(userConfig) {
   const { config, metaPlugins } = createServiceWorkerMetaConfig(userConfig);
   return applyPlugins(config, metaPlugins);
 }
 
+/**
+ * @param {BuildingRollupOptions} userConfig
+ */
 export function createServiceWorkerMetaConfig(userConfig = { output: {} }) {
   const developmentMode =
     typeof userConfig.developmentMode !== 'undefined'
@@ -31,14 +40,19 @@ export function createServiceWorkerMetaConfig(userConfig = { output: {} }) {
     },
   };
 
+  /**
+   * @type {import('plugins-manager').MetaPlugin<any>[]}
+   */
   let metaPlugins = [
     {
+      // @ts-ignore
       plugin: resolve,
       options: {
         moduleDirectories: ['node_modules', 'web_modules'],
       },
     },
     {
+      // @ts-ignore
       plugin: replace,
       options: {
         'process.env.NODE_ENV': JSON.stringify(developmentMode ? 'development' : 'production'),

packages/building-rollup/src/createSpaConfig.js

@@ -1,15 +1,26 @@
+// @ts-ignore
 import { rollupPluginHTML } from '@web/rollup-plugin-html';
+// @ts-ignore
 import { importMetaAssets } from '@web/rollup-plugin-import-meta-assets';
+// @ts-ignore
 import { polyfillsLoader } from '@web/rollup-plugin-polyfills-loader';
 import { applyPlugins } from 'plugins-manager';
 
 import { createBasicMetaConfig } from './createBasicConfig.js';
 
+/** @typedef {import('../types/main.js').BuildingRollupOptions} BuildingRollupOptions */
+
+/**
+ * @param {BuildingRollupOptions} [userConfig]
+ */
 export function createSpaConfig(userConfig) {
   const { config, metaPlugins } = createSpaMetaConfig(userConfig);
   return applyPlugins(config, metaPlugins);
 }
 
+/**
+ * @param {BuildingRollupOptions} userConfig
+ */
 export function createSpaMetaConfig(userConfig = { output: {} }) {
   const { config, metaPlugins, developmentMode } = createBasicMetaConfig(userConfig);
 
@@ -27,8 +38,13 @@ export function createSpaMetaConfig(userConfig = { output: {} }) {
   }
   delete config.absoluteBaseUrl;
 
+  /**
+   * @type {import('plugins-manager').MetaPlugin<any>[]}
+   */
   const spaMetaPlugins = [
+    // @ts-ignore
     ...metaPlugins,
+    // @ts-ignore
     {
       plugin: rollupPluginHTML,
       options: {
@@ -36,9 +52,11 @@ export function createSpaMetaConfig(userConfig = { output: {} }) {
         absoluteBaseUrl,
       },
     },
+    // @ts-ignore
     {
       plugin: importMetaAssets,
     },
+    // @ts-ignore
     {
       plugin: polyfillsLoader,
       options: {

packages/building-rollup/src/index.js

@@ -0,0 +1,11 @@
+/**
+ * @typedef {import('../types/main.js').BuildingRollupOptions} BuildingRollupOptions
+ */
+
+export { createBasicConfig, createBasicMetaConfig } from './createBasicConfig.js';
+export { createSpaConfig, createSpaMetaConfig } from './createSpaConfig.js';
+export { createMpaConfig, createMpaMetaConfig } from './createMpaConfig.js';
+export {
+  createServiceWorkerConfig,
+  createServiceWorkerMetaConfig,
+} from './createServiceWorkerConfig.js';

packages/building-rollup/test-node/createMpaConfig.test.js

@@ -1,6 +1,6 @@
 import chai from 'chai';
 import path from 'path';
-import fs from 'fs';
+import { readFile } from 'fs/promises';
 import { fileURLToPath } from 'url';
 import { rollup } from 'rollup';
 
@@ -8,7 +8,7 @@ const { expect } = chai;
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 /**
- * @param {object} config
+ * @param {import('@rocket/building-rollup').BuildingRollupOptions} config
  */
 async function buildAndWrite(config) {
   const bundle = await rollup(config);
@@ -16,21 +16,27 @@ async function buildAndWrite(config) {
   if (Array.isArray(config.output)) {
     await bundle.write(config.output[0]);
     await bundle.write(config.output[1]);
-  } else {
+  } else if (config.output) {
     await bundle.write(config.output);
   }
 }
 
+/**
+ * @param {string} configString
+ * @returns
+ */
 async function execute(configString) {
   const configPath = path.join(__dirname, 'fixtures', configString.split('/').join(path.sep));
   const config = (await import(configPath)).default;
   await buildAndWrite(config);
 
+  /**
+   * @param {string} fileName
+   */
   return async (fileName, { stripToBody = false, stripStartEndWhitespace = true } = {}) => {
-    let text = await fs.promises.readFile(
-      path.join(config.output.dir, fileName.split('/').join(path.sep)),
-    );
-    text = text.toString();
+    let text = (
+      await readFile(path.join(config.output.dir, fileName.split('/').join(path.sep)))
+    ).toString();
     if (stripToBody) {
       const bodyOpenTagEnd = text.indexOf('>', text.indexOf('<body') + 1) + 1;
       const bodyCloseTagStart = text.indexOf('</body>');

packages/building-rollup/test-node/spa-integration.test-revisit.js

@@ -1,10 +1,11 @@
-/* eslint-disable @typescript-eslint/ban-ts-comment */
 import puppeteer from 'puppeteer';
 import chai from 'chai';
 import path from 'path';
 import fs from 'fs';
+// @ts-ignore
 import rimraf from 'rimraf';
 import { rollup } from 'rollup';
+// @ts-ignore
 import { startDevServer } from '@web/dev-server';
 import { fileURLToPath } from 'url';
 
@@ -14,6 +15,7 @@ const rootDir = path.resolve(__dirname, '..', 'dist');
 const { expect } = chai;
 
 describe('spa integration tests', () => {
+  // @ts-ignore
   let server;
   /** @type {import('puppeteer').Browser} */
   let browser;
@@ -27,6 +29,7 @@ describe('spa integration tests', () => {
       readCliArgs: false,
       readFileConfig: false,
       logStartMessage: false,
+      // @ts-ignore
       clearTerminalOnReload: false,
     });
     browser = await puppeteer.launch();
@@ -35,6 +38,7 @@ describe('spa integration tests', () => {
 
   after(async () => {
     await browser.close();
+    // @ts-ignore
     await server.stop();
   });
 
@@ -45,6 +49,7 @@ describe('spa integration tests', () => {
   ].forEach(testCase => {
     describe(`testcase ${testCase}`, function describe() {
       this.timeout(30000);
+      // @ts-ignore
       let page;
 
       before(async () => {

packages/building-rollup/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../tsconfig.node-base.json",
+  "compilerOptions": {
+    "module": "ESNext",
+    "outDir": "./dist-types",
+    "rootDir": ".",
+    "composite": true,
+    "allowJs": true,
+    "checkJs": true,
+    "emitDeclarationOnly": true,
+    "moduleResolution": "NodeNext"
+  },
+  "include": ["src", "types", "test-node"],
+  "exclude": ["dist-types"]
+}

packages/building-rollup/types/main.d.ts

@@ -0,0 +1,8 @@
+import { RollupOptions } from 'rollup';
+
+interface BuildingRollupOptions extends RollupOptions {
+  developmentMode?: boolean;
+  rootDir?: string;
+  absoluteBaseUrl?: string;
+  setupPlugins?: function[];
+}

packages/check-html-links/CHANGELOG.md

@@ -1,5 +1,21 @@
 # check-html-links
 
+## 0.2.4
+
+### Patch Changes
+
+- 97d5fb2: Add external links validation via the flag `--validate-externals`.
+
+  You can/should provide an optional `--absolute-base-url` to handle urls starting with it as internal.
+
+  ```bash
+  # check external urls
+  npx check-html-links _site --validate-externals
+
+  # check external urls but treat links like https://rocket.modern-web.dev/about/ as internal
+  npx check-html-links _site --validate-externals --absolute-base-url https://rocket.modern-web.dev
+  ```
+
 ## 0.2.3
 
 ### Patch Changes

packages/check-html-links/README.md

@@ -14,7 +14,7 @@ npm i -D check-html-links
 npx check-html-links _site
 ```
 
-For docs please see our homepage [https://rocket.modern-web.dev/docs/tools/check-html-links/](https://rocket.modern-web.dev/docs/tools/check-html-links/).
+For docs please see our homepage [https://rocket.modern-web.dev/tools/check-html-links/overview/](https://rocket.modern-web.dev/tools/check-html-links/overview/).
 
 ## Comparison
 

packages/check-html-links/package.json

@@ -1,6 +1,6 @@
 {
   "name": "check-html-links",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "publishConfig": {
     "access": "public"
   },
@@ -37,6 +37,7 @@
     "command-line-args": "^5.1.1",
     "glob": "^7.0.0",
     "minimatch": "^3.0.4",
+    "node-fetch": "^3.0.0",
     "sax-wasm": "^2.0.0",
     "slash": "^4.0.0"
   },

packages/check-html-links/src/CheckHtmlLinksCli.js

@@ -1,12 +1,10 @@
-/* eslint-disable @typescript-eslint/ban-ts-comment */
-
 /** @typedef {import('../types/main').CheckHtmlLinksCliOptions} CheckHtmlLinksCliOptions */
 
 import path from 'path';
 import chalk from 'chalk';
 
 import commandLineArgs from 'command-line-args';
-import { validateFiles } from './validateFolder.js';
+import { prepareFiles, validateFiles } from './validateFolder.js';
 import { formatErrors } from './formatErrors.js';
 import { listFiles } from './listFiles.js';
 
@@ -18,7 +16,9 @@ export class CheckHtmlLinksCli {
     const mainDefinitions = [
       { name: 'ignore-link-pattern', type: String, multiple: true },
       { name: 'root-dir', type: String, defaultOption: true },
-      { name: 'continue-on-error', type: Boolean, defaultOption: false },
+      { name: 'continue-on-error', type: Boolean },
+      { name: 'validate-externals', type: Boolean },
+      { name: 'absolute-base-url', type: String },
     ];
     const options = commandLineArgs(mainDefinitions, {
       stopAtFirstUnknown: true,
@@ -29,6 +29,8 @@ export class CheckHtmlLinksCli {
       continueOnError: options['continue-on-error'],
       rootDir: options['root-dir'],
       ignoreLinkPatterns: options['ignore-link-pattern'],
+      validateExternals: options['validate-externals'],
+      absoluteBaseUrl: options['absolute-base-url'],
     };
   }
 
@@ -43,22 +45,47 @@ export class CheckHtmlLinksCli {
   }
 
   async run() {
-    const { ignoreLinkPatterns, rootDir: userRootDir } = this.options;
+    const {
+      ignoreLinkPatterns,
+      rootDir: userRootDir,
+      validateExternals,
+      absoluteBaseUrl,
+    } = this.options;
     const rootDir = userRootDir ? path.resolve(userRootDir) : process.cwd();
     const performanceStart = process.hrtime();
 
-    console.log('👀 Checking if all internal links work...');
     const files = await listFiles('**/*.html', rootDir);
 
+    console.log('Check HTML Links');
+
     const filesOutput =
       files.length == 0
-        ? '🧐 No files to check. Did you select the correct folder?'
-        : `🔥 Found a total of ${chalk.green.bold(files.length)} files to check!`;
+        ? '  🧐 No files to check. Did you select the correct folder?'
+        : `  📖 Found ${chalk.green.bold(files.length)} files containing`;
     console.log(filesOutput);
 
-    const { errors, numberLinks } = await validateFiles(files, rootDir, { ignoreLinkPatterns });
+    const { numberLinks, checkLocalFiles, checkExternalLinks } = await prepareFiles(
+      files,
+      rootDir,
+      {
+        ignoreLinkPatterns,
+        validateExternals,
+        absoluteBaseUrl,
+      },
+    );
 
-    console.log(`🔗 Found a total of ${chalk.green.bold(numberLinks)} links to validate!\n`);
+    console.log(`    🔗 ${chalk.green.bold(numberLinks)} internal links`);
+    if (validateExternals) {
+      console.log(`    🌐 ${chalk.green.bold(checkExternalLinks.length)} external links`);
+    }
+
+    console.log('  👀 Checking...');
+
+    const { errors } = await validateFiles({
+      checkLocalFiles,
+      validateExternals,
+      checkExternalLinks,
+    });
 
     const performance = process.hrtime(performanceStart);
     /** @type {string[]} */
@@ -70,7 +97,7 @@ export class CheckHtmlLinksCli {
         referenceCount += error.usage.length;
       }
       output = [
-        `❌ Found ${chalk.red.bold(
+        `  ❌ Found ${chalk.red.bold(
           errors.length.toString(),
         )} missing reference targets (used by ${referenceCount} links) while checking ${
           files.length
@@ -78,7 +105,7 @@ export class CheckHtmlLinksCli {
         ...formatErrors(errors)
           .split('\n')
           .map(line => `  ${line}`),
-        `Checking links duration: ${performance[0]}s ${performance[1] / 1000000}ms`,
+        `  🕑 Checking links duration: ${performance[0]}s ${performance[1] / 1000000}ms`,
       ];
       message = output.join('\n');
       if (this.options.printOnError === true) {
@@ -89,7 +116,7 @@ export class CheckHtmlLinksCli {
       }
     } else {
       console.log(
-        `✅ All internal links are valid. (executed in ${performance[0]}s ${
+        `  ✅ All tested links are valid. (executed in ${performance[0]}s ${
           performance[1] / 1000000
         }ms)`,
       );

packages/check-html-links/src/checkLinks.js

@@ -0,0 +1,63 @@
+import fetch from 'node-fetch';
+
+/**
+ * @type {Map<string,boolean>}
+ */
+const resultsMap = new Map();
+
+/**
+ *
+ * @param {string} url
+ * @param {boolean} result
+ * @returns {boolean}
+ */
+const memorizeCheckup = (url, result) => {
+  resultsMap.set(url, result);
+  return result;
+};
+
+/**
+ *
+ * @param {string} url
+ * @param {string} method
+ * @returns
+ */
+const fetchWrap = async (url, method = 'GET') => {
+  return Promise.race([
+    fetch(url, { method })
+      .then(response => response.ok)
+      .catch(() => false),
+    new Promise(resolve => setTimeout(resolve, 10000, false)),
+  ]);
+};
+
+/**
+ *
+ * @param {string} url
+ * @returns {Promise<boolean>}
+ */
+const fetchHead = async url => fetchWrap(url, 'HEAD');
+
+/**
+ *
+ * @param {string} url - URL object to check
+ * @returns {Promise<boolean>} true if url is alive or false if not
+ */
+const checkUrl = async url =>
+  (fetchHead(url) || fetchWrap(url)).then(result => memorizeCheckup(url, result));
+
+/**
+ *
+ * @param {string} link - link string to check
+ * @returns {Promise<boolean>}
+ */
+export const checkLink = async link => {
+  const url = link.startsWith('//') ? `https:${link}` : link;
+  return resultsMap.get(url) ?? checkUrl(url);
+};
+/**
+ * Check an array of links and return an object with
+ *
+ * @param {string[]} links Links to check
+ */
+export const checkLinks = async links => Promise.all(links.map(checkLink));

packages/check-html-links/src/formatErrors.js

@@ -15,7 +15,7 @@ export function formatErrors(errors, relativeFrom = process.cwd()) {
     const filePath = path.relative(relativeFrom, error.filePath);
     if (error.onlyAnchorMissing === true) {
       output.push(
-        `${number}. missing ${chalk.red.bold(
+        `  ${number}. missing ${chalk.red.bold(
           `id="${error.usage[0].anchor}"`,
         )} in ${chalk.cyanBright(filePath)}`,
       );
@@ -24,7 +24,7 @@ export function formatErrors(errors, relativeFrom = process.cwd()) {
       const title =
         firstAttribute === 'src' || firstAttribute === 'srcset' ? 'file' : 'reference target';
 
-      output.push(`${number}. missing ${title} ${chalk.red.bold(filePath)}`);
+      output.push(`  ${number}. missing ${title} ${chalk.red.bold(filePath)}`);
     }
     const usageLength = error.usage.length;
 
@@ -34,11 +34,11 @@ export function formatErrors(errors, relativeFrom = process.cwd()) {
       const clickAbleLink = chalk.cyanBright(`${usagePath}:${usage.line + 1}:${usage.character}`);
       const attributeStart = chalk.gray(`${usage.attribute}="`);
       const attributeEnd = chalk.gray('"');
-      output.push(`  from ${clickAbleLink} via ${attributeStart}${usage.value}${attributeEnd}`);
+      output.push(`    from ${clickAbleLink} via ${attributeStart}${usage.value}${attributeEnd}`);
     }
     if (usageLength > 3) {
       const more = chalk.red((usageLength - 3).toString());
-      output.push(`  ... ${more} more references to this target`);
+      output.push(`    ... ${more} more references to this target`);
     }
     output.push('');
   }

packages/check-html-links/src/validateFolder.js

@@ -1,12 +1,11 @@
-/* eslint-disable @typescript-eslint/ban-ts-comment */
 import fs from 'fs';
 import saxWasm from 'sax-wasm';
 import minimatch from 'minimatch';
 import { createRequire } from 'module';
-
-import { listFiles } from './listFiles.js';
 import path from 'path';
 import slash from 'slash';
+import { listFiles } from './listFiles.js';
+import { checkLinks } from './checkLinks.js';
 
 /** @typedef {import('../types/main').Link} Link */
 /** @typedef {import('../types/main').LocalFile} LocalFile */
@@ -28,6 +27,9 @@ const parserIds = new SAXParser(SaxEventType.Attribute, streamOptions);
 /** @type {Error[]} */
 let checkLocalFiles = [];
 
+/** @type {Error[]} */
+let checkExternalLinks = [];
+
 /** @type {Error[]} */
 let errors = [];
 
@@ -151,6 +153,26 @@ function addLocalFile(filePath, anchor, usageObj) {
   }
 }
 
+/**
+ * @param {string} filePath
+ * @param {Usage} usageObj
+ */
+function addExternalLink(filePath, usageObj) {
+  const foundIndex = checkExternalLinks.findIndex(item => {
+    return item.filePath === filePath;
+  });
+
+  if (foundIndex === -1) {
+    checkExternalLinks.push({
+      filePath,
+      onlyAnchorMissing: false,
+      usage: [usageObj],
+    });
+  } else {
+    checkExternalLinks[foundIndex].usage.push(usageObj);
+  }
+}
+
 /**
  * @param {string} inValue
  */
@@ -200,11 +222,16 @@ function isNonHttpSchema(url) {
  * @param {object} options
  * @param {string} options.htmlFilePath
  * @param {string} options.rootDir
+ * @param {string} options.absoluteBaseUrl
  * @param {function(string): boolean} options.ignoreUsage
  */
-async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage }) {
+async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage, absoluteBaseUrl }) {
   for (const hrefObj of links) {
-    const { value, anchor } = getValueAndAnchor(hrefObj.value);
+    const { value: rawValue, anchor } = getValueAndAnchor(hrefObj.value);
+
+    const value = rawValue.startsWith(absoluteBaseUrl)
+      ? rawValue.substring(absoluteBaseUrl.length)
+      : rawValue;
 
     const usageObj = {
       attribute: hrefObj.attribute,
@@ -229,8 +256,7 @@ async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage }) {
     } else if (valueFile === '' && anchor !== '') {
       addLocalFile(htmlFilePath, anchor, usageObj);
     } else if (value.startsWith('//') || value.startsWith('http')) {
-      // TODO: handle external urls
-      // external url - we do not handle that (yet)
+      addExternalLink(htmlFilePath, usageObj);
     } else if (value.startsWith('/')) {
       const filePath = path.join(rootDir, valueFile);
       addLocalFile(filePath, anchor, usageObj);
@@ -244,7 +270,7 @@ async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage }) {
     }
   }
 
-  return { checkLocalFiles: [...checkLocalFiles] };
+  return { checkLocalFiles: [...checkLocalFiles], checkExternalLinks: [...checkExternalLinks] };
 }
 
 /**
@@ -283,17 +309,34 @@ async function validateLocalFiles(checkLocalFiles) {
   return errors;
 }
 
+/**
+ *
+ * @param {Error[]} checkExternalLinks
+ */
+async function validateExternalLinks(checkExternalLinks) {
+  for await (const localFileObj of checkExternalLinks) {
+    const links = localFileObj.usage.map(usage => usage.value);
+    const results = await checkLinks(links);
+    localFileObj.usage = localFileObj.usage.filter((link, index) => !results[index]);
+    if (localFileObj.usage.length > 0) {
+      errors.push(localFileObj);
+    }
+  }
+  return errors;
+}
+
 /**
  * @param {string[]} files
  * @param {string} rootDir
  * @param {Options} opts?
  */
-export async function validateFiles(files, rootDir, opts) {
+export async function prepareFiles(files, rootDir, opts) {
   await parserReferences.prepareWasm(saxWasmBuffer);
   await parserIds.prepareWasm(saxWasmBuffer);
 
   errors = [];
   checkLocalFiles = [];
+  checkExternalLinks = [];
   idCache = new Map();
   let numberLinks = 0;
 
@@ -309,11 +352,27 @@ export async function validateFiles(files, rootDir, opts) {
   for (const htmlFilePath of files) {
     const { links } = await extractReferences(htmlFilePath);
     numberLinks += links.length;
-    await resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage });
+    await resolveLinks(links, {
+      htmlFilePath,
+      rootDir,
+      ignoreUsage,
+      absoluteBaseUrl: opts?.absoluteBaseUrl,
+    });
   }
-  await validateLocalFiles(checkLocalFiles);
+  return { checkLocalFiles, checkExternalLinks, numberLinks };
+}
 
-  return { errors: errors, numberLinks: numberLinks };
+/**
+ * @param {*} param0
+ * @returns
+ */
+export async function validateFiles({ checkLocalFiles, validateExternals, checkExternalLinks }) {
+  await validateLocalFiles(checkLocalFiles);
+  if (validateExternals) {
+    await validateExternalLinks(checkExternalLinks);
+  }
+
+  return { errors };
 }
 
 /**
@@ -323,6 +382,14 @@ export async function validateFiles(files, rootDir, opts) {
 export async function validateFolder(inRootDir, opts) {
   const rootDir = path.resolve(inRootDir);
   const files = await listFiles('**/*.html', rootDir);
-  const { errors } = await validateFiles(files, rootDir, opts);
+
+  const { checkLocalFiles, checkExternalLinks } = await prepareFiles(files, rootDir, opts);
+
+  const { errors } = await validateFiles({
+    checkLocalFiles,
+    validateExternals: opts?.validateExternals,
+    checkExternalLinks,
+  });
+
   return errors;
 }

packages/check-html-links/test-node/fixtures/external-link/index.html

@@ -1,5 +1,9 @@
-<!-- ignore known subsystems -->
-<a href="/docs/"></a>
-<a href="/developer/getting-started.html#js"></a>
-<a href="/developer/language-guides/"></a>
-<a href="/developer/javascript.html"></a>
+<!-- valid -->
+<a href="//rocket.modern-web.dev/"></a>
+<a href="http://rocket.modern-web.dev/"></a>
+<a href="https://rocket.modern-web.dev/"></a>
+
+<!-- invalid -->
+<a href="//rocket.modern-web.dev/unexists-page/"></a>
+<a href="http://rocket.modern-web.dev/unexists-page/"></a>
+<a href="https://rocket.modern-web.dev/unexists-page/"></a>

packages/check-html-links/test-node/fixtures/internal-link/index.html

@@ -5,8 +5,5 @@
 <a href="./page.html"></a>
 <a href=" ./page.html "></a>
 <a href=" /page.html "></a>
-<a href="//domain.com/something/"></a>
-<a href="http://domain.com/something/"></a>
-<a href="https://domain.com/something/"></a>
 <a href=""></a>
 <a href=":~:text=put%20your%20labels%20above%20your%20inputs">Sign-in form best practices</a>

packages/check-html-links/test-node/fixtures/internal-own-absolute-base-path/about.html

@@ -0,0 +1 @@
+About

packages/check-html-links/test-node/fixtures/internal-own-absolute-base-path/index.html

@@ -0,0 +1,2 @@
+<a href="about.html">About</a>
+<a href="http://localhost/about.html">About Absolute</a>

packages/check-html-links/test-node/formatErrors.test.js

@@ -20,19 +20,19 @@ describe('formatErrors', () => {
     const result = await executeAndFormat('fixtures/test-case');
     expect(result.trim().split('\n')).to.deep.equal([
       '1. missing id="my-teams" in fixtures/test-case/price/index.html',
-      '  from fixtures/test-case/history/index.html:1:9 via href="/price/#my-teams"',
+      '    from fixtures/test-case/history/index.html:1:9 via href="/price/#my-teams"',
       '',
-      '2. missing file fixtures/test-case/about/images/team.png',
-      '  from fixtures/test-case/about/index.html:3:10 via src="./images/team.png"',
+      '  2. missing file fixtures/test-case/about/images/team.png',
+      '    from fixtures/test-case/about/index.html:3:10 via src="./images/team.png"',
       '',
-      '3. missing reference target fixtures/test-case/aboot',
-      '  from fixtures/test-case/about/index.html:6:11 via href="/aboot"',
-      '  from fixtures/test-case/history/index.html:4:11 via href="/aboot"',
-      '  from fixtures/test-case/index.html:4:11 via href="/aboot"',
-      '  ... 2 more references to this target',
+      '  3. missing reference target fixtures/test-case/aboot',
+      '    from fixtures/test-case/about/index.html:6:11 via href="/aboot"',
+      '    from fixtures/test-case/history/index.html:4:11 via href="/aboot"',
+      '    from fixtures/test-case/index.html:4:11 via href="/aboot"',
+      '    ... 2 more references to this target',
       '',
-      '4. missing reference target fixtures/test-case/prce',
-      '  from fixtures/test-case/index.html:1:9 via href="./prce"',
+      '  4. missing reference target fixtures/test-case/prce',
+      '    from fixtures/test-case/index.html:1:9 via href="./prce"',
     ]);
   });
 });

packages/check-html-links/test-node/validateFolder.test.js

@@ -66,6 +66,74 @@ describe('validateFolder', () => {
     ]);
   });
 
+  it('validates external links', async () => {
+    const { errors, cleanup } = await execute('fixtures/external-link', {
+      validateExternals: true,
+    });
+    expect(cleanup(errors)).to.deep.equal([
+      {
+        filePath: 'fixtures/external-link/index.html',
+        onlyAnchorMissing: false,
+        usage: [
+          {
+            attribute: 'href',
+            value: '//rocket.modern-web.dev/unexists-page/',
+            file: 'fixtures/external-link/index.html',
+            line: 6,
+            character: 9,
+            anchor: '',
+          },
+          {
+            attribute: 'href',
+            value: 'http://rocket.modern-web.dev/unexists-page/',
+            file: 'fixtures/external-link/index.html',
+            line: 7,
+            character: 9,
+            anchor: '',
+          },
+          {
+            attribute: 'href',
+            value: 'https://rocket.modern-web.dev/unexists-page/',
+            file: 'fixtures/external-link/index.html',
+            line: 8,
+            character: 9,
+            anchor: '',
+          },
+        ],
+      },
+    ]);
+  });
+
+  it('validates links with own absolute base url as internal', async () => {
+    const { errors, cleanup } = await execute('fixtures/internal-own-absolute-base-path', {
+      validateExternals: true,
+      absoluteBaseUrl: 'http://localhost',
+    });
+    expect(cleanup(errors)).to.deep.equal([]);
+  });
+
+  it('validates all full urls if there is no absoluteBaseUrl provided', async () => {
+    const { errors, cleanup } = await execute('fixtures/internal-own-absolute-base-path', {
+      validateExternals: true,
+    });
+    expect(cleanup(errors)).to.deep.equal([
+      {
+        filePath: 'fixtures/internal-own-absolute-base-path/index.html',
+        onlyAnchorMissing: false,
+        usage: [
+          {
+            anchor: '',
+            attribute: 'href',
+            character: 9,
+            file: 'fixtures/internal-own-absolute-base-path/index.html',
+            line: 1,
+            value: 'http://localhost/about.html',
+          },
+        ],
+      },
+    ]);
+  });
+
   it('groups multiple usage of the same missing file', async () => {
     const { errors, cleanup } = await execute('fixtures/internal-links-to-same-file');
     expect(cleanup(errors)).to.deep.equal([

packages/check-html-links/types/main.d.ts

@@ -26,13 +26,15 @@ export interface Error {
   usage: Usage[];
 }
 
-interface Options {
+export interface Options {
   ignoreLinkPatterns: string[] | null;
+  validateExternals: boolean;
+  absoluteBaseUrl: string;
 }
 
-export interface CheckHtmlLinksCliOptions {
+export interface CheckHtmlLinksCliOptions extends Options {
   printOnError: boolean;
   rootDir: string;
-  ignoreLinkPatterns: string[] | null;
   continueOnError: boolean;
+  absoluteBaseUrl: string;
 }

packages/check-website/CHANGELOG.md

@@ -0,0 +1,87 @@
+# check-html-links
+
+## 0.2.4
+
+### Patch Changes
+
+- 97d5fb2: Add external links validation via the flag `--validate-externals`.
+
+  You can/should provide an optional `--absolute-base-url` to handle urls starting with it as internal.
+
+  ```bash
+  # check external urls
+  npx check-html-links _site --validate-externals
+
+  # check external urls but treat links like https://rocket.modern-web.dev/about/ as internal
+  npx check-html-links _site --validate-externals --absolute-base-url https://rocket.modern-web.dev
+  ```
+
+## 0.2.3
+
+### Patch Changes
+
+- 5043429: Ignore `<a href="tel:9999">` links
+- f08f926: Add missing `slash` dependency
+- a0e8edf: Ignore links containing not http schema urls like `sketch://`, `vscode://`, ...
+
+  ```html
+  <a href="sketch://add-library?url=https%3A%2F%2Fmyexample.com%2Fdesign%2Fui-kit.xml"></a>
+  <a href="vscode://file/c:/myProject/package.json:5:10"></a>
+  ```
+
+- 1949b1e: Ignore plain and html encoded mailto links
+
+  ```html
+  <!-- source -->
+  <a href="mailto:address@example.com">contact</a>
+
+  <!-- html encoded -->
+  <a
+    href="&#109;&#97;&#105;&#108;&#116;&#111;&#58;&#97;&#100;&#100;&#114;&#101;&#115;&#115;&#64;&#101;&#120;&#97;&#109;&#112;&#108;&#101;&#46;&#99;&#111;&#109;"
+    >contact</a
+  >
+  ```
+
+## 0.2.2
+
+### Patch Changes
+
+- 66c2d78: fix: windows path issue
+
+## 0.2.1
+
+### Patch Changes
+
+- be0d0b3: fix: add missing main entry to the packages
+
+## 0.2.0
+
+### Minor Changes
+
+- 750418b: Uses a class for the CLI and adding the following options:
+
+  - `--root-dir` the root directory to serve files from. Defaults to the current working directory
+  - `--ignore-link-pattern` do not check links matching the pattern
+  - `--continue-on-error` if present it will not exit with an error code - useful while writing or for temporary passing a ci
+
+  BREAKING CHANGE:
+
+  - Exists with an error code if a broken link is found
+
+## 0.1.2
+
+### Patch Changes
+
+- f343c50: When reading bigger files, especially bigger files with all content on one line it could mean a read chunk is in the middle of a character. This can lead to strange symbols in the resulting string. The `hightWaterMark` is now increased from the node default of 16KB to 256KB. Additionally, the `hightWaterMark` is now synced for reading and parsing.
+
+## 0.1.1
+
+### Patch Changes
+
+- eb74110: Add info about how many files and links will be checked
+
+## 0.1.0
+
+### Minor Changes
+
+- cd22231: Initial release

packages/check-website/README.md

@@ -0,0 +1,28 @@
+# Check Website
+
+A fast checker for broken links/references in HTML.
+
+## Installation
+
+```shell
+npm i -D check-website
+```
+
+## Usage
+
+```bash
+npx check-website
+```
+
+For docs please see our homepage [https://rocket.modern-web.dev/tools/check-html-links/overview/](https://rocket.modern-web.dev/tools/check-html-links/overview/).
+
+## Comparison
+
+Checking the output of the [11ty-website](https://github.com/11ty/11ty-website) with 13 missing reference targets (used by 516 links) while checking 501 files. (on January 17, 2021)
+
+| Tool             | Lines printed | Times  | Lang | Dependency Tree |
+| ---------------- | ------------- | ------ | ---- | --------------- |
+| check-html-links | 38            | ~2.5s  | node | 19              |
+| link-checker     | 3000+         | ~11s   | node | 106             |
+| hyperlink        | 68            | 4m 20s | node | 481             |
+| htmltest         | 1000+         | ~0.7s  | GO   | -               |

packages/check-website/TODO.md

@@ -0,0 +1,3 @@
+# Check HTML Links
+
+- Support external entrypoints... e.g. user tried this `npx check-html-links@latest https://jasik.xyz`

packages/check-website/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "check-website",
+  "version": "0.0.1",
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "A fast low dependency checker of html links/references",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/modernweb-dev/rocket.git",
+    "directory": "packages/check-website"
+  },
+  "author": "Modern Web <hello@modern-web.dev> (https://modern-web.dev/)",
+  "homepage": "https://rocket.modern-web.dev/docs/tools/check-website/",
+  "bin": {
+    "check-website": "src/cli.js"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist-types/index.d.ts",
+      "default": "./src/index.js"
+    }
+  },
+  "scripts": {
+    "test": "mocha --timeout 5000 test-node/**/*.test.js test-node/*.test.js",
+    "test:watch": "onchange 'src/**/*.js' 'test-node/**/*.js' -- npm test",
+    "types": "wireit"
+  },
+  "files": [
+    "dist-types",
+    "src"
+  ],
+  "dependencies": {
+    "cacheable-lookup": "^7.0.0",
+    "colorette": "^2.0.16",
+    "commander": "^9.0.0",
+    "got": "^12.5.2",
+    "mime-types": "^2.1.35",
+    "minimatch": "^3.0.4",
+    "node-fetch": "^3.0.0",
+    "normalize-url": "^7.2.0",
+    "p-queue": "^7.3.0",
+    "sax-wasm": "^2.0.0",
+    "slash": "^4.0.0"
+  },
+  "devDependencies": {
+    "@types/mime-types": "^2.1.1"
+  },
+  "wireit": {
+    "types": {
+      "command": "tsc --build --pretty",
+      "dependencies": [
+        "../plugins-manager:types"
+      ],
+      "files": [
+        "src/**/*.js",
+        "test-node/**/*.js",
+        "types",
+        "tsconfig.json"
+      ],
+      "output": [
+        "dist-types/**"
+      ]
+    }
+  }
+}

packages/check-website/src/CheckWebsiteCli.js

@@ -0,0 +1,217 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment */
+
+/** @typedef {import('../types/main.js').FullCheckWebsiteCliOptions} FullCheckWebsiteCliOptions */
+/** @typedef {import('../types/main.js').CheckWebsiteCliOptions} CheckWebsiteCliOptions */
+/** @typedef {import('./assets/Asset.js').Asset} Asset */
+
+import { EventEmitter } from 'events';
+import path from 'path';
+import { Command } from 'commander';
+import { green } from 'colorette';
+import { gatherFiles } from './helpers/gatherFiles.js';
+import { renderProgressBar } from './cli/renderProgressBar.js';
+import { LocalReferencesPlugin } from './plugins/LocalReferencesPlugin.js';
+// import { HasCanonicalPlugin } from './plugins/HasCanonicalPlugin.js';
+import { ExternalReferencesPlugin } from './plugins/ExternalReferencesPlugin.js';
+import { AssetManager } from './assets/AssetManager.js';
+import { LitTerminal } from './cli/LitTerminal.js';
+import { cli } from './cli/cli.js';
+import { IssueManager } from './issues/IssueManager.js';
+import { hr } from './cli/helpers.js';
+import { existsSync } from 'fs';
+import { readFile } from 'fs/promises';
+import { pathToFileURL } from 'url';
+import { normalizeUrl } from './helpers/normalizeUrl.js';
+import { HtmlPage } from './assets/HtmlPage.js';
+
+export class CheckWebsiteCli extends LitTerminal {
+  /** @type {FullCheckWebsiteCliOptions} */
+  options = {
+    configFile: '',
+    inputDir: process.cwd(),
+    originUrl: '',
+    assetManager: new AssetManager(),
+    issueManager: new IssueManager(),
+    plugins: [],
+  };
+
+  /**
+   * @readonly
+   */
+  events = new EventEmitter();
+
+  constructor({ argv = process.argv } = {}) {
+    super();
+    this.argv = argv;
+
+    this.program = new Command();
+    this.program.allowUnknownOption().option('-c, --config-file <path>', 'path to config file');
+    this.program.parse(this.argv);
+
+    this.program.allowUnknownOption(false);
+
+    if (this.program.opts().configFile) {
+      this.options.configFile = this.program.opts().configFile;
+    }
+
+    this.program
+      .option('-i, --input-dir <path>', 'path to where to search for source files')
+      .action(async cliOptions => {
+        this.setOptions(cliOptions);
+      });
+
+    this.options.plugins = [
+      //
+      new LocalReferencesPlugin(),
+      // new HasCanonicalPlugin(),
+      new ExternalReferencesPlugin(),
+    ];
+
+    /** @param {string} msg */
+    this.options.issueManager.logger = msg => this.logStatic(msg);
+  }
+
+  /**
+   * @param {Partial<CheckWebsiteCliOptions>} newOptions
+   */
+  setOptions(newOptions) {
+    this.options = {
+      ...this.options,
+      ...newOptions,
+    };
+  }
+
+  async applyConfigFile() {
+    if (this.options.configFile) {
+      const configFilePath = path.resolve(this.options.configFile);
+      const fileOptions = (await import(configFilePath)).default;
+      this.setOptions(fileOptions);
+    } else {
+      // make sure all default settings are properly initialized
+      this.setOptions({});
+    }
+  }
+
+  async execute() {
+    super.execute();
+    await this.applyConfigFile();
+    // const inputDir = userInputDir ? path.resolve(userInputDir) : process.cwd();
+
+    let entrypoint = path.join(this.options.inputDir, 'index.html');
+    if (!existsSync(entrypoint)) {
+      console.log(`Entrypoint ${entrypoint} does not exist`);
+      process.exit(1);
+    }
+    if (!this.options.originUrl) {
+      const entryHtml = await readFile(entrypoint, 'utf-8');
+      const canonicalUrl = findCanonicalUrl(entryHtml);
+      if (canonicalUrl) {
+        this.options.originUrl = canonicalUrl;
+      } else {
+        console.log(`No canonical url found in ${entrypoint}`);
+        process.exit(1);
+      }
+    }
+
+    if (!this.options.isLocalUrl) {
+      /** @param {string} url */
+      this.options.isLocalUrl = url => {
+        const normalizedUrl = normalizeUrl(url);
+        return normalizedUrl.startsWith(this.options.originUrl);
+      };
+    }
+
+    this.logStatic('🔎 Check Website');
+    this.logStatic('');
+    this.logStatic(`👀 Start crawling from ${green('index.html')}`);
+    this.logStatic(`📄 Will follow all links within ${green(this.options.originUrl)}`);
+    this.logStatic('');
+
+    const onParseElementCallbacks = this.options.plugins
+      .map(plugin => plugin.onParseElement)
+      .filter(Boolean);
+
+    this.options.assetManager = new AssetManager({
+      originPath: this.options.inputDir,
+      originUrl: this.options.originUrl,
+      // TODO: fix type...
+      // @ts-ignore
+      onParseElementCallbacks,
+      plugins: this.options.plugins,
+      isLocalUrl: this.options.isLocalUrl,
+      skips: this.options.skips,
+    });
+    const pluginsAndAssetManager = [...this.options.plugins, this.options.assetManager];
+
+    this.options.assetManager.events.on('idle', () => {
+      if (pluginsAndAssetManager.every(p => p.isIdle)) {
+        this.updateComplete.then(() => {
+          this.events.emit('done');
+        });
+      }
+    });
+
+    for (const plugin of this.options.plugins) {
+      plugin.assetManager = this.options.assetManager;
+      plugin.issueManager = this.options.issueManager;
+      plugin.isLocalUrl = this.options.isLocalUrl;
+      await plugin.setup(this);
+      plugin.events.on('progress', () => {
+        this.requestUpdate();
+      });
+      plugin.events.on('idle', () => {
+        if (pluginsAndAssetManager.every(p => p.isIdle)) {
+          this.updateComplete.then(() => {
+            this.events.emit('done');
+          });
+        }
+      });
+    }
+
+    const files = await gatherFiles(this.options.inputDir);
+    if (files.length === 0) {
+      console.log('🧐 No files to check. Did you select the correct folder?');
+      process.exit(1);
+    }
+
+    for (const [, file] of files.entries()) {
+      const fileUrl = pathToFileURL(file);
+      this.options.assetManager.addExistingFile(fileUrl);
+    }
+
+    // start crawling at the main index.html
+    const rootPage = this.options.assetManager.get(this.options.originUrl + '/index.html');
+    if (rootPage && rootPage instanceof HtmlPage) {
+      await rootPage.parse();
+    }
+  }
+
+  renderParsing() {
+    const title = `Parsing:`.padEnd(11);
+    const total = this.options.assetManager.parsingQueue.getTotal();
+    const doneNr = this.options.assetManager.parsingQueue.getDone();
+    const duration = this.options.assetManager.parsingQueue.getDuration();
+    const progress = renderProgressBar(doneNr, 0, total);
+    const minNumberLength = `${total}`.length;
+    const done = `${doneNr}`.padStart(minNumberLength);
+    return `${title} ${progress} ${done}/${total} files | 🕑 ${duration}s`;
+  }
+
+  render() {
+    return cli`
+      \n${hr()}
+      ${this.renderParsing()}
+      ${this.options.plugins.map(plugin => plugin.render()).join('\n')}
+    `;
+  }
+}
+
+/**
+ * @param {string} html
+ */
+function findCanonicalUrl(html) {
+  const matches = html.match(/<link\s*rel="canonical"\s*href="(.*)"/);
+  if (matches) {
+    return normalizeUrl(matches[1]);
+  }
+}

packages/check-website/src/assets/Asset.js

@@ -0,0 +1,138 @@
+import { EventEmitter } from 'events';
+import fetch from 'node-fetch';
+import got, { RequestError } from 'got';
+import CacheableLookup from 'cacheable-lookup';
+
+const dnsCache = new CacheableLookup();
+
+/** @typedef {import('../../types/main.js').AssetStatus} AssetStatus */
+
+export const ASSET_STATUS = /** @type {const} */ ({
+  unknown: 0,
+
+  // 200+ exists
+  exists: 200,
+  existsLocal: 220,
+  existsExternal: 230,
+
+  // 250+ parsed
+  parsed: 250,
+  parsedLocal: 260,
+  parsedExternal: 270,
+
+  // 300+ redirect
+  redirect: 300,
+
+  // 400+ error
+  missing: 400,
+});
+
+export class Asset {
+  /**
+   * The full page URL.
+   *
+   * Example: `https://docs.astro.build/en/getting-started/`
+   * @type {URL}
+   */
+  url;
+
+  /**
+   * @type {AssetStatus}
+   */
+  #status = ASSET_STATUS.unknown;
+
+  /**
+   * @param {AssetStatus} status
+   */
+  set status(status) {
+    this.#status = status;
+    this.events.emit('status-changed');
+  }
+
+  get status() {
+    return this.#status;
+  }
+
+  /**
+   * @readonly
+   */
+  events = new EventEmitter();
+
+  localPath = '';
+
+  localSourcePath = '';
+
+  /** @type {import('../../types/main.js').FullAssetOptions} */
+  options = {
+    originUrl: '',
+    originPath: '',
+    localPath: '',
+    localSourcePath: '',
+    fetch,
+    assetManager: undefined,
+    isLocalUrl: url => url.startsWith(this.options.originUrl),
+    skip: false,
+  };
+
+  /**
+   *
+   * @param {URL} url
+   * @param {import('../../types/main.js').AssetOptions} options
+   */
+  constructor(url, options) {
+    this.options = { ...this.options, ...options };
+    this.url = url;
+    this.localPath = this.options.localPath;
+
+    if (this.url.protocol === 'file:') {
+      throw new Error(`File protocol is not supported. Used by ${this.url.href}`);
+    }
+  }
+
+  exists() {
+    return new Promise(resolve => {
+      if (this.status > ASSET_STATUS.unknown && this.status < ASSET_STATUS.missing) {
+        resolve(true);
+      } else if (this.options.isLocalUrl(this.url.href)) {
+        // Local assets need to be added upfront and are not dynamically discovered - potentially a feature for later
+        resolve(false);
+      } else {
+        this.options.assetManager?.fetchQueue
+          .add(async () => await this.executeExists())
+          .finally(() => {
+            resolve(this.status > ASSET_STATUS.unknown && this.status < ASSET_STATUS.missing);
+          });
+      }
+    });
+  }
+
+  async executeExists() {
+    if (this.options.skip) {
+      return;
+    }
+    // TODO: detect server redirects (301, 302, etc)?
+    // const fetching = fetch(this.url.href, { method: 'HEAD', redirect: "error" });
+    try {
+      await got(this.url.href, {
+        method: 'HEAD',
+        retry: {
+          limit: 1,
+          calculateDelay: ({ computedValue }) => {
+            return computedValue / 100;
+          },
+        },
+        dnsCache,
+      });
+      this.status = ASSET_STATUS.existsExternal;
+    } catch (err) {
+      if (err instanceof RequestError) {
+        this.status =
+          /** @type {AssetStatus} */ (err?.response?.statusCode) || ASSET_STATUS.missing;
+      }
+    }
+  }
+
+  isLocal() {
+    return this.status === ASSET_STATUS.existsLocal || this.status === ASSET_STATUS.parsedLocal;
+  }
+}

packages/check-website/src/assets/AssetManager.js

@@ -0,0 +1,220 @@
+import fetch from 'node-fetch';
+import path from 'path';
+import { fileURLToPath, pathToFileURL } from 'url';
+import mime from 'mime-types';
+
+import { Asset, ASSET_STATUS } from './Asset.js';
+import { HtmlPage } from './HtmlPage.js';
+import { normalizeUrl, normalizeToLocalUrl } from '../helpers/normalizeUrl.js';
+import { decodeNumberHtmlEntities } from '../helpers/decodeNumberHtmlEntities.js';
+import { Queue } from '../helpers/Queue.js';
+import EventEmitter from 'events';
+import minimatch from 'minimatch';
+
+/** @typedef {import('../plugins/Plugin.js').Plugin} Plugin */
+
+const classMap = {
+  Asset,
+  HtmlPage,
+};
+
+/**
+ * @param {string} url
+ * @returns {URL}
+ */
+function getUrl(url) {
+  try {
+    return new URL(url);
+  } catch (e) {
+    // handle html encoded mailto links like <a href="&#109;&#97;&#105;&#108;&#116;&#111;&#58;">
+    const decoded = decodeNumberHtmlEntities(url);
+    return new URL(decoded);
+  }
+}
+
+export class AssetManager {
+  /** @type {Map<string, Asset | HtmlPage>} */
+  assets = new Map();
+
+  /**
+   * @readonly
+   */
+  events = new EventEmitter();
+
+  /** Queue *************************/
+  parsingQueue = new Queue({
+    concurrency: 1,
+  });
+
+  fetchQueue = new Queue({
+    concurrency: 1,
+    carryoverConcurrencyCount: true,
+    interval: 500,
+    intervalCap: 10,
+  });
+
+  /** @type {import('../../types/main.js').FullAssetManagerOptions} */
+  options = {
+    originUrl: '',
+    originPath: '',
+    fetch,
+    plugins: [],
+    isLocalUrl: url => url.startsWith(this.options.originUrl),
+    onParseElementCallbacks: [],
+  };
+
+  /**
+   * @param {import('../../types/main.js').AssetManagerOptions} [options]
+   */
+  constructor(options) {
+    this.options = { ...this.options, ...options };
+
+    this.parsingQueue.on('idle', () => {
+      this.events.emit('idle');
+    });
+    this.fetchQueue.on('idle', () => {
+      this.events.emit('idle');
+    });
+  }
+
+  /**
+   * @param {string} localPath
+   * @returns {Asset | HtmlPage}
+   */
+  addFile(localPath) {
+    const url = pathToFileURL(localPath); // new URL('file://' + localPath);
+    return this.addUrl(url);
+  }
+
+  /**
+   * It does not check if the asset actually exits.
+   * ONLY call it for asset urls you know exist.
+   *
+   * @param {URL} fileUrl
+   * @return {Asset | HtmlPage}
+   */
+  addExistingFile(fileUrl) {
+    const filePath = fileURLToPath(fileUrl);
+    const rel = path.relative(this.options.originPath, filePath);
+    const url = new URL(rel, this.options.originUrl);
+    const localPath = path.join(this.options.originPath, rel);
+    if (this.has(url.href)) {
+      const found = this.get(url.href);
+      if (found) {
+        return found;
+      }
+    }
+
+    const mimeType = mime.lookup(fileUrl.pathname);
+    const skip = this.options.skips && this.options.skips.some(skip => minimatch(url.href, skip));
+
+    /** @type {keyof classMap} */
+    let typeClass = 'Asset';
+    if (mimeType === 'text/html') {
+      typeClass = 'HtmlPage';
+    }
+    const asset = new classMap[typeClass](url, {
+      assetManager: this,
+      localPath,
+      originPath: this.options.originPath,
+      originUrl: this.options.originUrl,
+      onParseElementCallbacks: this.options.onParseElementCallbacks,
+      skip,
+    });
+    asset.status = ASSET_STATUS.existsLocal;
+    this.assets.set(this.normalizeUrl(url.href), asset);
+
+    for (const plugin of this.options.plugins) {
+      plugin.onNewParsedAsset(asset);
+    }
+
+    return asset;
+  }
+
+  /**
+   * @param {URL | string} url
+   * @param {{ mimeType?: string }} options
+   * @returns {Asset | HtmlPage}
+   */
+  addUrl(url, { mimeType = '' } = {}) {
+    const useUrl = typeof url === 'string' ? getUrl(url) : url;
+    if (this.has(useUrl.href)) {
+      return /** @type {Asset | HtmlPage} */ (this.get(useUrl.href));
+    }
+
+    const skip =
+      this.options.skips && this.options.skips.some(skip => minimatch(useUrl.href, skip));
+
+    /** @type {keyof classMap} */
+    let typeClass = 'Asset';
+    if (mimeType === 'text/html') {
+      typeClass = 'HtmlPage';
+    }
+    const asset = new classMap[typeClass](useUrl, {
+      assetManager: this,
+      originPath: this.options.originPath,
+      originUrl: this.options.originUrl,
+      onParseElementCallbacks: this.options.onParseElementCallbacks,
+      skip,
+    });
+
+    this.assets.set(this.normalizeUrl(useUrl.href), asset);
+
+    for (const plugin of this.options.plugins) {
+      plugin.onNewParsedAsset(asset);
+    }
+
+    return asset;
+  }
+
+  /**
+   * @param {string} url
+   * @returns {string}
+   */
+  normalizeUrl(url) {
+    if (this.options.isLocalUrl(url)) {
+      return normalizeToLocalUrl(url);
+    }
+    return normalizeUrl(url);
+  }
+
+  /**
+   * @param {string} url
+   * @returns
+   */
+  get(url) {
+    return this.assets.get(this.normalizeUrl(url));
+  }
+
+  /**
+   * @param {string} url
+   * @returns {boolean}
+   */
+  has(url) {
+    return this.assets.has(this.normalizeUrl(url));
+  }
+
+  get size() {
+    return this.assets.size;
+  }
+
+  /**
+   * @param {string} url
+   * @returns {Asset | HtmlPage}
+   */
+  getAsset(url) {
+    let asset = this.get(url);
+    if (!asset) {
+      asset = this.addUrl(new URL(url));
+    }
+    return asset;
+  }
+
+  getAll() {
+    return Array.from(this.assets.values());
+  }
+
+  get isIdle() {
+    return this.fetchQueue.isIdle && this.parsingQueue.isIdle;
+  }
+}

packages/check-website/src/assets/HtmlPage.js

@@ -0,0 +1,246 @@
+import fs from 'fs';
+
+import { Asset, ASSET_STATUS } from './Asset.js';
+
+import {
+  getAttributeInfo,
+  getLinksFromSrcSet,
+  resolveToFullPageUrl,
+} from '../helpers/sax-helpers.js';
+import { parser, SaxEventType, streamOptions } from '../helpers/sax-parser.js';
+
+/** @typedef {import('sax-wasm').Tag} Tag */
+/** @typedef {import('../../types/main.js').Reference} Reference */
+
+export class HtmlPage extends Asset {
+  /**
+   * @type {Reference[]}
+   */
+  references = [];
+
+  /**
+   * A list of hashes that can be used as URL fragments to jump to specific parts of the page.
+   * @type {string[]}
+   */
+  hashes = [];
+
+  /**
+   * The target URL of a `<meta http-equiv="refresh" content="...">` element
+   * contained on the page (if any).
+   * @type {URL | undefined}
+   */
+  redirectTargetUrl;
+
+  referenceSources = [
+    { tagName: 'img', attribute: 'src' },
+    { tagName: 'img', attribute: 'srcset' },
+    { tagName: 'source', attribute: 'src' },
+    { tagName: 'source', attribute: 'srcset' },
+    { tagName: 'a', attribute: 'href' },
+    { tagName: 'link', attribute: 'href' },
+    { tagName: 'script', attribute: 'src' },
+  ];
+
+  /** @type {import('../../types/main.js').FullHtmlPageOptions} */
+  options = {
+    ...this.options,
+    onParseElementCallbacks: [],
+  };
+
+  /**
+   * @param {URL} url
+   * @param {import('../../types/main.js').HtmlPageOptions} options
+   */
+  constructor(url, options) {
+    super(url, {
+      onParseElementCallbacks: [],
+      ...options,
+    });
+    this.options = { ...this.options, ...options };
+  }
+
+  /**
+   * @param {string} hash
+   * @returns {boolean}
+   */
+  hasHash(hash) {
+    const checkHash = hash.startsWith('#') ? hash.substring(1) : hash;
+    return this.hashes.includes(checkHash);
+  }
+
+  async parse() {
+    if (!(await this.exists())) {
+      return;
+    }
+
+    return await this._parse();
+  }
+
+  #parsing = false;
+  #parsingPromise = Promise.resolve(false);
+
+  /**
+   *
+   * @protected
+   * @returns {Promise<boolean>}
+   */
+  _parse() {
+    if (this.#parsing) {
+      return this.#parsingPromise;
+    }
+
+    this.#parsing = true;
+    this.#parsingPromise = new Promise(resolve => {
+      if (this.status >= ASSET_STATUS.parsed && this.status < ASSET_STATUS.missing) {
+        this.#parsing = false;
+        resolve(true);
+      } else {
+        if (!this.options.assetManager) {
+          throw new Error('You need to pass an assetManager to the options');
+        }
+        this.options.assetManager?.parsingQueue
+          .add(async () => await this.executeParse())
+          .then(() => {
+            this.#parsing = false;
+            resolve(true);
+          });
+      }
+    });
+    return this.#parsingPromise;
+  }
+
+  async executeParse() {
+    if (this.options.skip) {
+      return;
+    }
+    if (!(await this.exists())) {
+      return;
+    }
+    await this._executeParse();
+  }
+
+  /**
+   *
+   * @protected
+   * @returns {Promise<void>}
+   */
+  _executeParse() {
+    parser.eventHandler = (ev, _data) => {
+      if (ev === SaxEventType.CloseTag) {
+        const data = /** @type {Tag} */ (/** @type {any} */ (_data));
+
+        const searchTags = this.referenceSources.map(({ tagName }) => tagName);
+        if (searchTags.includes(data.name)) {
+          const possibleAttributes = this.referenceSources
+            .map(({ attribute, tagName }) => (tagName === data.name ? attribute : undefined))
+            .filter(Boolean);
+          for (const possibleAttributeName of possibleAttributes) {
+            if (possibleAttributeName) {
+              const attribute = getAttributeInfo(data, possibleAttributeName);
+              if (attribute) {
+                const { value, start, end, name } = attribute;
+                /** @type {Reference} */
+                const entry = {
+                  start,
+                  end,
+                  value,
+                  url: resolveToFullPageUrl(this.url.href, value),
+                  page: this,
+                  attribute: name,
+                  tag: data.name,
+                };
+                if (name === 'srcset') {
+                  const links = getLinksFromSrcSet(value, this.url.href, entry);
+                  this.references.push(...links);
+                } else {
+                  this.references.push(entry);
+                  if (this.status === ASSET_STATUS.existsLocal) {
+                    // only add "sub" assets for local files
+                    this.options.assetManager?.addUrl(entry.url);
+                  }
+                }
+              }
+            }
+          }
+        }
+
+        const idData = getAttributeInfo(data, 'id');
+        if (idData) {
+          this.hashes.push(idData.value);
+        }
+
+        if (data.name === 'a') {
+          const nameData = getAttributeInfo(data, 'name');
+          if (nameData) {
+            this.hashes.push(nameData.value);
+          }
+        }
+
+        /** @type {import('../../types/main.js').ParseElement} */
+        const element = {
+          tagName: data.name.toUpperCase(),
+          getAttribute: name => getAttributeInfo(data, name)?.value,
+        };
+        this.options.onParseElementCallbacks.forEach(cb => cb(element, this));
+
+        // Check if the page redirects somewhere else using meta refresh
+        if (data.name === 'meta') {
+          const httpEquivData = getAttributeInfo(data, 'http-equiv');
+          if (httpEquivData && httpEquivData.value.toLowerCase() === 'refresh') {
+            const metaRefreshContent = getAttributeInfo(data, 'content')?.value;
+            const metaRefreshMatches = metaRefreshContent?.match(
+              /^([0-9]+)\s*;\s*url\s*=\s*(.+)$/i,
+            );
+            this.redirectTargetUrl = metaRefreshMatches
+              ? new URL(metaRefreshMatches[2], this.url.href)
+              : undefined;
+
+            if (this.status === ASSET_STATUS.existsLocal && this.redirectTargetUrl) {
+              // only add "sub" assets for local files
+              this.options.assetManager?.addUrl(this.redirectTargetUrl.href, {
+                mimeType: 'text/html',
+              });
+            }
+          }
+        }
+      }
+    };
+
+    return new Promise((resolve, reject) => {
+      if (this.status === ASSET_STATUS.existsLocal) {
+        if (!this.options.localPath) {
+          throw new Error(`Missing local path on the asset ${this.url.href}`);
+        }
+        // Read from FileSystem
+        const readable = fs.createReadStream(this.options.localPath, streamOptions);
+        readable.on('data', chunk => {
+          // @ts-expect-error
+          parser.write(chunk);
+        });
+        readable.on('end', () => {
+          parser.end();
+          this.status = ASSET_STATUS.parsedLocal;
+          resolve();
+        });
+      }
+
+      if (this.status === ASSET_STATUS.existsExternal) {
+        // Fetch from the network
+        this.options.fetch(this.url.href).then(async response => {
+          if (!response.ok || !response.body) {
+            reject('Error in response');
+            return;
+          }
+          response.body.on('data', chunk => {
+            parser.write(chunk);
+          });
+          response.body.on('end', () => {
+            parser.end();
+            this.status = ASSET_STATUS.parsedExternal;
+            resolve();
+          });
+        });
+      }
+    });
+  }
+}

packages/check-website/src/cli.js

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+
+import { existsSync } from 'fs';
+import path from 'path';
+import { CheckWebsiteCli } from './CheckWebsiteCli.js';
+
+const cli = new CheckWebsiteCli();
+
+const cwd = process.cwd();
+const configFiles = [
+  path.join('config', 'check-website.config.js'),
+  path.join('config', 'check-website.config.mjs'),
+  'check-website.config.js',
+  'check-website.config.mjs',
+  path.join('..', 'config', 'check-website.config.js'),
+  path.join('..', 'config', 'check-website.config.mjs'),
+  path.join('..', 'check-website.config.js'),
+  path.join('..', 'check-website.config.mjs'),
+];
+
+for (const configFile of configFiles) {
+  const configFilePath = path.join(cwd, configFile);
+  if (existsSync(configFilePath)) {
+    cli.options.configFile = configFilePath;
+    break;
+  }
+}
+
+await cli.execute();

packages/check-website/src/cli/LitTerminal.js

@@ -0,0 +1,383 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment */
+import { hideCursor } from './helpers.js';
+
+const terminal = process.stderr;
+
+export class LitTerminal {
+  dynamicRender = true;
+
+  /**
+   * They dynamic cli view.
+   * It can only render as many lines as the terminal height which is usually around 24-40.
+   * If you write more dynamic lines, an error will be thrown.
+   * To display more information you can use logStatic() to render lines that will not by dynamic.
+   *
+   * Why?
+   * If you write more lines than the terminal height then scrolling will happen and at this point it is
+   * no longer accessible by the terminal itself.
+   * Scrolling is a feature of terminal simulators, not the terminal itself.
+   * This means that once content got scrolled out of the terminal, it is no longer adjustable.
+   *
+   * @example
+   *   render() {
+   *     return cli`
+   *       Counter: ${this.counter}
+   *     `;
+   *   }
+   *
+   * @returns {string}
+   */
+  render() {
+    return '';
+  }
+
+  constructor() {
+    /** @type {typeof LitTerminal} */ (this.constructor).__finalize();
+  }
+
+  /**
+   * Log a static string that is rendered above the dynamic view.
+   *
+   * Use it to display information like a list of errors or completed tasks.
+   * This content will not be cleared when the dynamic view is rendered and will always be reachable via scrolling.
+   *
+   * @param {string} message
+   */
+  logStatic(message) {
+    this.#staticLogs.push(message);
+    this.requestUpdate();
+  }
+
+  /**
+   * Request a rerender of the dynamic view.
+   * You can call this as often as you want and it will only rerender once per LitTerminal.renderInterval.
+   * Typically you don't need to call this manually as it is called automatically when you update a property or use logStatic().
+   * You can use it to create your own getters/setters
+   * @example
+   *   _counter = 0;
+   *   set counter(counter) {
+   *     this._counter = counter;
+   *     this.requestUpdate();
+   *   }
+   *   get counter() {
+   *     return this._counter;
+   *   }s
+   */
+  requestUpdate() {
+    if (this.#updateRequested === false) {
+      this.#updateRequested = true;
+      setTimeout(() => {
+        this._render();
+        this.#updateRequested = false;
+      }, /** @type {typeof LitTerminal} */ (this.constructor).renderInterval);
+    }
+  }
+
+  /**
+   * This actually
+   */
+  execute() {
+    hideCursor();
+  }
+
+  //
+  // End of public API
+  //
+
+  #lastRender = '';
+
+  /** @type {string[]} */
+  #staticLogs = [];
+
+  #updateRequested = false;
+
+  /**
+   * How often (in ms) the dynamic view should be rerendered [defaults to 100]
+   */
+  static renderInterval = 100;
+
+  /**
+   * This writes the result of render() & logStatic() to the terminal.
+   * It compares the last render with the current render and only clears and writes the changed lines.
+   *
+   * It throws an error you render() tries to write more lines than the terminal height.
+   */
+  _render() {
+    if (this.dynamicRender === false) {
+      for (const staticLog of this.#staticLogs) {
+        console.log(staticLog);
+      }
+      this.#staticLogs = [];
+      return;
+    }
+
+    const render = this.render();
+    const renderLines = render.split('\n');
+
+    const windowSize = terminal.getWindowSize();
+    if (renderLines.length > windowSize[1]) {
+      throw new Error(
+        `You rendered ${renderLines.length} lines while the terminal height is only ${windowSize[1]}. For non dynamic parts use logStatic()`,
+      );
+    }
+
+    if (render !== this.#lastRender || this.#staticLogs.length) {
+      const lastRenderLines = this.#lastRender.split('\n');
+      if (lastRenderLines.length > 0) {
+        terminal.moveCursor(0, lastRenderLines.length * -1);
+        terminal.cursorTo(0);
+      }
+
+      const staticLength = this.#staticLogs.length;
+      if (staticLength) {
+        for (const staticLog of this.#staticLogs) {
+          terminal.clearLine(0);
+          console.log(staticLog);
+        }
+        this.#staticLogs = [];
+      }
+
+      for (const [index, line] of renderLines.entries()) {
+        if (line.length !== lastRenderLines[index - staticLength]?.length) {
+          terminal.clearLine(0);
+        }
+        console.log(line);
+      }
+      terminal.clearScreenDown();
+
+      this.#lastRender = render;
+    }
+  }
+
+  //
+  // ***********************************************************************************************
+  // Below if inspired by ReactiveElement
+  // https://github.com/lit/lit/blob/main/packages/reactive-element/src/reactive-element.ts
+  //
+
+  static finalized = false;
+  static properties = {};
+
+  /**
+   * Creates property accessors for registered properties, sets up element
+   * styling, and ensures any superclasses are also finalized. Returns true if
+   * the element was finalized.
+   * @nocollapse
+   */
+  static __finalize() {
+    if (this.finalized) {
+      return false;
+    }
+    this.finalized = true;
+    // // finalize any superclasses
+    // const superCtor = Object.getPrototypeOf(this);
+    // superCtor.finalize();
+
+    if (this.properties) {
+      const props = this.properties;
+      // support symbols in properties (IE11 does not support this)
+      const propKeys = [
+        ...Object.getOwnPropertyNames(props),
+        ...Object.getOwnPropertySymbols(props),
+      ];
+      for (const p of propKeys) {
+        // @ts-ignore
+        this.createProperty(p, props[p]);
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Creates a property accessor on the element prototype if one does not exist
+   * and stores a {@linkcode PropertyDeclaration} for the property with the
+   * given options. The property setter calls the property's `hasChanged`
+   * property option or uses a strict identity check to determine whether or not
+   * to request an update.
+   *
+   * This method may be overridden to customize properties; however,
+   * when doing so, it's important to call `super.createProperty` to ensure
+   * the property is setup correctly. This method calls
+   * `getPropertyDescriptor` internally to get a descriptor to install.
+   * To customize what properties do when they are get or set, override
+   * `getPropertyDescriptor`. To customize the options for a property,
+   * implement `createProperty` like this:
+   *
+   * ```ts
+   * static createProperty(name, options) {
+   *   options = Object.assign(options, {myOption: true});
+   *   super.createProperty(name, options);
+   * }
+   * ```
+   *
+   * @nocollapse
+   * @category properties
+   * @param {PropertyKey} name Name of property
+   * @param {import('../../types/LitTerminal.js').PropertyDeclaration} options Property declaration
+   */
+  static createProperty(name, options) {
+    // Note, since this can be called by the `@property` decorator which
+    // is called before `finalize`, we ensure finalization has been kicked off.
+    this.__finalize();
+
+    this.elementProperties.set(name, options);
+
+    // Do not generate an accessor if the prototype already has one, since
+    // it would be lost otherwise and that would never be the user's intention;
+    // Instead, we expect users to call `requestUpdate` themselves from
+    // user-defined accessors. Note that if the super has an accessor we will
+    // still overwrite it
+    // eslint-disable-next-line no-prototype-builtins
+    if (!options.noAccessor && !this.prototype.hasOwnProperty(name)) {
+      const key = typeof name === 'symbol' ? Symbol() : `__${name}`;
+      const descriptor = this.getPropertyDescriptor(name, key, options);
+      if (descriptor !== undefined) {
+        Object.defineProperty(this.prototype, name, descriptor);
+      }
+    }
+  }
+
+  /**
+   * Returns a property descriptor to be defined on the given named property.
+   * If no descriptor is returned, the property will not become an accessor.
+   * For example,
+   *
+   * ```ts
+   * class MyElement extends LitElement {
+   *   static getPropertyDescriptor(name, key, options) {
+   *     const defaultDescriptor =
+   *         super.getPropertyDescriptor(name, key, options);
+   *     const setter = defaultDescriptor.set;
+   *     return {
+   *       get: defaultDescriptor.get,
+   *       set(value) {
+   *         setter.call(this, value);
+   *         // custom action.
+   *       },
+   *       configurable: true,
+   *       enumerable: true
+   *     }
+   *   }
+   * }
+   * ```
+   *
+   * @nocollapse
+   * @category properties
+   * @param {PropertyKey} name
+   * @param {string | symbol} key
+   * @param {import('../../types/LitTerminal.js').PropertyDeclaration} options
+   * @returns {PropertyDescriptor | undefined}
+   */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  static getPropertyDescriptor(name, key, options) {
+    return {
+      /**
+       * @this {LitTerminal}
+       */
+      get() {
+        // @ts-ignore
+        return this[key];
+        // return (this as {[key: string]: unknown})[key as string];
+      },
+      /**
+       * @param {unknown} value
+       * @this {LitTerminal}
+       */
+      set(value) {
+        // const oldValue = this[name];
+        // @ts-ignore
+        this[key] = value;
+        this.requestUpdate();
+        // this.requestUpdate(name, oldValue, options);
+      },
+      configurable: true,
+      enumerable: true,
+    };
+  }
+
+  static elementProperties = new Map();
+
+  /**
+   * Returns the property options associated with the given property.
+   * These options are defined with a `PropertyDeclaration` via the `properties`
+   * object or the `@property` decorator and are registered in
+   * `createProperty(...)`.
+   *
+   * Note, this method should be considered "final" and not overridden. To
+   * customize the options for a given property, override
+   * {@linkcode createProperty}.
+   *
+   * @nocollapse
+   * @final
+   * @category properties
+   * @param {string | symbol} name
+   */
+  static getPropertyOptions(name) {
+    return this.elementProperties.get(name) || defaultPropertyDeclaration;
+  }
+
+  /**
+   * Returns a Promise that resolves when the element has completed updating.
+   * The Promise value is a boolean that is `true` if the element completed the
+   * update without triggering another update. The Promise result is `false` if
+   * a property was set inside `updated()`. If the Promise is rejected, an
+   * exception was thrown during the update.
+   *
+   * To await additional asynchronous work, override the `getUpdateComplete`
+   * method. For example, it is sometimes useful to await a rendered element
+   * before fulfilling this Promise. To do this, first await
+   * `super.getUpdateComplete()`, then any subsequent state.
+   *
+   * @category updates
+   * @returns {Promise<boolean>} A promise of a boolean that resolves to true if the update completed
+   *   without triggering another update.
+   */
+  get updateComplete() {
+    return new Promise(resolve => setTimeout(resolve, 100));
+    // TODO: implement actual waiting for a finished render
+    // return this.getUpdateComplete();
+  }
+
+  // /**
+  //  * Override point for the `updateComplete` promise.
+  //  *
+  //  * It is not safe to override the `updateComplete` getter directly due to a
+  //  * limitation in TypeScript which means it is not possible to call a
+  //  * superclass getter (e.g. `super.updateComplete.then(...)`) when the target
+  //  * language is ES5 (https://github.com/microsoft/TypeScript/issues/338).
+  //  * This method should be overridden instead. For example:
+  //  *
+  //  * ```ts
+  //  * class MyElement extends LitElement {
+  //  *   override async getUpdateComplete() {
+  //  *     const result = await super.getUpdateComplete();
+  //  *     await this._myChild.updateComplete;
+  //  *     return result;
+  //  *   }
+  //  * }
+  //  * ```
+  //  *
+  //  * @returns {Promise<boolean>} A promise of a boolean that resolves to true if the update completed
+  //  *     without triggering another update.
+  //  * @category updates
+  //  */
+  // getUpdateComplete() {
+  //   return this.__updatePromise;
+  // }
+}
+
+const defaultPropertyDeclaration = {
+  type: String,
+  hasChanged: notEqual,
+};
+
+/**
+ * Change function that returns true if `value` is different from `oldValue`.
+ * This method is used as the default for a property's `hasChanged` function.
+ * @param {unknown} value The new value
+ * @param {unknown} old The old value
+ */
+export function notEqual(value, old) {
+  // This ensures (old==NaN, value==NaN) always returns false
+  return old !== value && (old === old || value === value);
+}

packages/check-website/src/cli/cli.js

@@ -0,0 +1,72 @@
+// TODO: support nested lists with new lines like: ${items.map(item => cli`[ ] ${item}\n`)}
+// will most likely require a stateful "cli tagged template literal" that passes on the current indent level
+
+/**
+ * Tagged template literal that
+ * - dedents the string
+ * - removes empty first/last lines
+ * - joins arrays
+ *
+ * @example
+ *   const str = cli`
+ *     Welcome ${name}!
+ *     List: ${items.map(item => `[ ] ${item} `)}
+ *   `;
+ *   # becomes
+ *   Welcome John!
+ *   List: [ ] a [ ] b [ ] c
+ *
+ * @param {TemplateStringsArray} strings
+ * @param  {...any} values
+ * @returns
+ */
+export function cli(strings, ...values) {
+  const useStrings = typeof strings === 'string' ? [strings] : strings.raw;
+  let result = '';
+
+  for (var i = 0; i < useStrings.length; i++) {
+    let currentString = useStrings[i];
+    currentString
+      // join lines when there is a suppressed newline
+      .replace(/\\\n[ \t]*/g, '')
+      // handle escaped backticks
+      .replace(/\\`/g, '`');
+    result += currentString;
+
+    if (i < values.length) {
+      const value = Array.isArray(values[i]) ? values[i].join('') : values[i];
+      result += value;
+    }
+  }
+
+  // now strip indentation
+  const lines = result.split('\n');
+  let minIndent = -1;
+  for (const line of lines) {
+    const match = line.match(/^(\s+)\S+/);
+    if (match) {
+      const indent = match[1].length;
+      if (minIndent === -1) {
+        // this is the first indented line
+        minIndent = indent;
+      } else {
+        minIndent = Math.min(minIndent, indent);
+      }
+    }
+  }
+
+  let finalResult = '';
+  if (minIndent !== -1) {
+    for (const line of lines) {
+      finalResult += line[0] === ' ' ? line.slice(minIndent) : line;
+      finalResult += '\n';
+    }
+  }
+
+  return (
+    finalResult
+      .trim()
+      // handle escaped newlines at the end to ensure they don't get stripped too
+      .replace(/\\n/g, '\n')
+  );
+}

packages/check-website/src/cli/helpers.js

@@ -0,0 +1,11 @@
+export function hr(length = process.stderr.getWindowSize()[0]) {
+  return '─'.repeat(length);
+}
+
+export function hideCursor() {
+  process.stderr.write('\u001B[?25l');
+}
+
+export function showCursor() {
+  process.stderr.write('\u001B[?25h');
+}

packages/check-website/src/cli/renderProgressBar.js

@@ -0,0 +1,40 @@
+import { gray, white } from 'colorette';
+
+const PROGRESS_BLOCKS = [' ', '▏', '▎', '▍', '▌', '▋', '▊', '▉', '█'];
+const PROGRESS_WIDTH = 30;
+
+/**
+ *
+ * @param {number} value
+ * @param {number} total
+ * @returns
+ */
+function createProgressBlocks(value, total) {
+  if (value >= total) {
+    return PROGRESS_BLOCKS[8].repeat(PROGRESS_WIDTH);
+  }
+
+  const count = (PROGRESS_WIDTH * value) / total;
+  const floored = Math.floor(count);
+  const partialBlock =
+    PROGRESS_BLOCKS[Math.floor((count - floored) * (PROGRESS_BLOCKS.length - 1))];
+  return `${PROGRESS_BLOCKS[8].repeat(floored)}${partialBlock}${' '.repeat(
+    PROGRESS_WIDTH - floored - 1,
+  )}`;
+}
+
+/**
+ *
+ * @param {number} finished
+ * @param {number} active
+ * @param {number} total
+ * @returns
+ */
+export function renderProgressBar(finished, active, total) {
+  const progressBlocks = createProgressBlocks(finished + active, total);
+  const finishedBlockCount = Math.floor((PROGRESS_WIDTH * finished) / total);
+
+  const finishedBlocks = white(progressBlocks.slice(0, finishedBlockCount));
+  const scheduledBlocks = gray(progressBlocks.slice(finishedBlockCount));
+  return `|${finishedBlocks}${scheduledBlocks}|`;
+}

packages/check-website/src/helpers/Queue.js

@@ -0,0 +1,50 @@
+import { formatPerformance } from './formatPerformance.js';
+import PQueue from 'p-queue';
+
+/** @typedef {import('p-queue').QueueAddOptions} QueueAddOptions */
+/** @typedef {import('p-queue').PriorityQueue} PriorityQueue  */
+
+export class Queue extends PQueue {
+  #total = 0;
+
+  /** @type {[number, number] | undefined} */
+  #durationStart;
+  /** @type {[number, number] | undefined} */
+  duration;
+
+  isIdle = true;
+
+  /**
+   * @param {import('p-queue').Options<PriorityQueue, QueueAddOptions>} [options]
+   */
+  constructor(options) {
+    super(options);
+    this.on('active', () => {
+      if (!this.#durationStart) {
+        this.#durationStart = process.hrtime();
+      }
+      this.isIdle = false;
+    });
+    this.on('completed', () => {
+      this.duration = process.hrtime(this.#durationStart);
+    });
+    this.on('idle', () => {
+      this.isIdle = true;
+    });
+    this.on('add', () => {
+      this.#total += 1;
+    });
+  }
+
+  getDuration() {
+    return this.duration ? formatPerformance(this.duration) : '0.00';
+  }
+
+  getDone() {
+    return this.#total - this.size - this.pending;
+  }
+
+  getTotal() {
+    return this.#total;
+  }
+}

packages/check-website/src/helpers/decodeNumberHtmlEntities.js

@@ -0,0 +1,15 @@
+/**
+ * Converts number HTML entities to their corresponding characters.
+ *
+ * Example:
+ * mailto: => mailto:
+ *
+ * @param {string} str
+ * @returns {string}
+ */
+export function decodeNumberHtmlEntities(str) {
+  return str.replace(/&#([0-9]{1,3});/gi, (match, numStr) => {
+    const num = parseInt(numStr, 10); // read num as normal number
+    return String.fromCharCode(num);
+  });
+}

packages/check-website/src/helpers/formatPerformance.js

@@ -0,0 +1,7 @@
+/**
+ * @param {[number, number]} perf
+ * @returns {string}
+ */
+export function formatPerformance(perf) {
+  return (perf[0] + perf[1] / 1e9).toFixed(2);
+}

packages/check-website/src/helpers/gatherFiles.js

@@ -0,0 +1,45 @@
+import { readdir } from 'fs/promises';
+import path from 'path';
+
+/**
+ * @param {string} fileName
+ * @returns {boolean}
+ */
+function isIndexFile(fileName) {
+  return fileName === 'index.html';
+}
+
+/**
+ * @param {string | URL} inRootDir
+ * @returns {Promise<string[]>}
+ */
+export async function gatherFiles(inRootDir) {
+  const rootDir = inRootDir instanceof URL ? inRootDir.pathname : path.resolve(inRootDir);
+  let files = [];
+
+  const entries = await readdir(rootDir, { withFileTypes: true });
+
+  // 1. handle possible index.html file
+  for (const entry of entries) {
+    if (!entry.isDirectory() && isIndexFile(entry.name)) {
+      files.push(path.join(rootDir, entry.name));
+    }
+  }
+  // 2. handle other html files
+  for (const entry of entries) {
+    const { name } = entry;
+    if (entry.isFile() && !isIndexFile(name)) {
+      const filePath = path.join(rootDir, name);
+      files.push(filePath);
+    }
+  }
+  // 3. handle sub directories
+  for (const entry of entries) {
+    if (entry.isDirectory()) {
+      const currentPath = path.join(rootDir, entry.name);
+      files.push(...(await gatherFiles(currentPath)));
+    }
+  }
+
+  return files;
+}

packages/check-website/src/helpers/normalizeUrl.js

@@ -0,0 +1,42 @@
+import normalizeUrlDep from 'normalize-url';
+
+const normalizeOptions = {
+  stripAuthentication: true,
+  stripHash: true,
+  stripTextFragment: true,
+  removeQueryParameters: false,
+  // removeTrailingSlash: false,
+  // removeSingleSlash: false,
+  removeDirectoryIndex: true,
+  // removeExplicitPort: true,
+};
+
+/**
+ * @param {string} url
+ * @param {import('normalize-url').Options} options
+ * @returns {string}
+ */
+export function normalizeUrl(url, options = {}) {
+  if (url.startsWith('mailto:')) {
+    return url.toLowerCase();
+  }
+  // = "mailto:" but html encoded)
+  if (url.startsWith('mailto:')) {
+    return url;
+  }
+  if (url.startsWith('tel:')) {
+    return url.toLowerCase();
+  }
+  if (url.startsWith('about:')) {
+    return url.toLowerCase();
+  }
+  return normalizeUrlDep(url, { ...normalizeOptions, ...options }).toLowerCase();
+}
+
+/**
+ * @param {string} url
+ * @returns {string}
+ */
+export function normalizeToLocalUrl(url) {
+  return normalizeUrl(url, { removeQueryParameters: true });
+}

packages/check-website/src/helpers/sax-helpers.js

@@ -0,0 +1,66 @@
+/** @typedef {import('sax-wasm').Tag} Tag */
+
+/**
+ * @param {Tag} data
+ * @param {string} name
+ */
+export function getAttributeInfo(data, name) {
+  if (data.attributes) {
+    const { attributes } = data;
+    const foundIndex = attributes.findIndex(entry => entry.name.value === name);
+    if (foundIndex !== -1) {
+      const entry = attributes[foundIndex].value;
+      return {
+        value: entry.value,
+        start: `${entry.start.line + 1}:${entry.start.character}`,
+        end: `${entry.end.line + 1}:${entry.end.character}`,
+        name,
+      };
+    }
+  }
+  return undefined;
+}
+
+/**
+ * @param {string} pageUrl
+ * @param {string} referenceUrl
+ * @returns {string}
+ */
+export function resolveToFullPageUrl(pageUrl, referenceUrl) {
+  // = "mailto:" but html encoded)
+  if (referenceUrl.startsWith('mailto:')) {
+    return referenceUrl;
+  }
+  if (referenceUrl.startsWith('about:')) {
+    return referenceUrl;
+  }
+  const url = new URL(referenceUrl, pageUrl);
+  return url.href;
+}
+
+/**
+ * @param {string} value Comma
+ * @param {string} pageUrl
+ * @param {import('../../types/main.js').Reference} entry
+ * @returns
+ */
+export function getLinksFromSrcSet(value, pageUrl, entry) {
+  const links = [];
+  if (value.includes(',')) {
+    const srcsets = value.split(',').map(el => el.trim());
+    for (const srcset of srcsets) {
+      if (srcset.includes(' ')) {
+        const srcsetParts = srcset.split(' ');
+        links.push({ ...entry, url: resolveToFullPageUrl(pageUrl, srcsetParts[0]) });
+      } else {
+        links.push({ ...entry, url: resolveToFullPageUrl(pageUrl, srcset) });
+      }
+    }
+  } else if (value.includes(' ')) {
+    const srcsetParts = value.split(' ');
+    links.push({ ...entry, url: resolveToFullPageUrl(pageUrl, srcsetParts[0]) });
+  } else {
+    links.push(entry);
+  }
+  return links;
+}

packages/check-website/src/helpers/sax-parser.js

@@ -0,0 +1,14 @@
+import saxWasm from 'sax-wasm';
+import { createRequire } from 'module';
+import { readFile } from 'fs/promises';
+
+export const { SaxEventType, SAXParser } = saxWasm;
+
+const require = createRequire(import.meta.url);
+
+export const streamOptions = { highWaterMark: 256 * 1024 };
+const saxPath = require.resolve('sax-wasm/lib/sax-wasm.wasm');
+const saxWasmBuffer = await readFile(saxPath);
+export const parser = new SAXParser(SaxEventType.CloseTag, streamOptions);
+
+await parser.prepareWasm(saxWasmBuffer);

packages/check-website/src/index.js

@@ -0,0 +1,14 @@
+export { normalizeToLocalUrl, normalizeUrl } from './helpers/normalizeUrl.js';
+
+export { CheckWebsiteCli } from './CheckWebsiteCli.js';
+
+export { LocalReferencesPlugin } from './plugins/LocalReferencesPlugin.js';
+export { ExternalReferencesPlugin } from './plugins/ExternalReferencesPlugin.js';
+export { HasCanonicalPlugin } from './plugins/HasCanonicalPlugin.js';
+
+export { Asset, ASSET_STATUS } from './assets/Asset.js';
+export { HtmlPage } from './assets/HtmlPage.js';
+export { AssetManager } from './assets/AssetManager.js';
+
+
+/** @typedef {import('../types/main.js').CheckWebsiteCliOptions} CheckWebsiteCliOptions */

packages/check-website/src/issues/Issue.js

@@ -0,0 +1,33 @@
+export class Issue {
+  options = {
+    sortOrder: 0,
+    duplicate: false,
+    filePath: '',
+    title: 'Issue',
+    message: '',
+    icon: '❌',
+    logger: console.log,
+    page: null,
+    skip: false,
+  };
+
+  /**
+   * @param {Partial<{}>} options
+   */
+  constructor(options = {}) {
+    this.options = { ...this.options, ...options };
+  }
+
+  /**
+   * @param {(msg: string) => void} logger
+   * @returns {void}
+   */
+  render(logger) {
+    const useLogger = logger || this.options.logger;
+    if (this.options.duplicate) {
+      return;
+    }
+    useLogger(`${this.options.icon} ${this.options.title}: ${this.options.message}`);
+    useLogger(`  🛠️  ${this.options.filePath}`);
+  }
+}

packages/check-website/src/issues/IssueManager.js

@@ -0,0 +1,24 @@
+export class IssueManager {
+  renderErrorsOnAdd = true;
+  logger = console.log;
+
+  /**
+   * @type {(import('./Issue.js').Issue | import('./PageIssue.js').PageIssue | import('./ReferenceIssue.js').ReferenceIssue)[]}
+   */
+  issues = [];
+
+  /**
+   *
+   * @param {import('./Issue.js').Issue | import('./PageIssue.js').PageIssue | import('./ReferenceIssue.js').ReferenceIssue} issue
+   */
+  add(issue) {
+    this.issues.push(issue);
+    if (this.renderErrorsOnAdd) {
+      issue.render(this.logger);
+    }
+  }
+
+  all() {
+    return this.issues;
+  }
+}

packages/check-website/src/issues/PageIssue.js

@@ -0,0 +1,11 @@
+import { Issue } from './Issue.js';
+
+export class PageIssue extends Issue {
+  constructor(options = {}) {
+    super({
+      message: '',
+      title: 'Page Issue',
+      ...options,
+    });
+  }
+}

packages/check-website/src/issues/ReferenceIssue.js

@@ -0,0 +1,11 @@
+import { Issue } from './Issue.js';
+
+export class ReferenceIssue extends Issue {
+  constructor(options = {}) {
+    super({
+      title: 'Not Found',
+      icon: '🔗',
+      ...options,
+    });
+  }
+}

packages/check-website/src/plugins/ExternalReferencesPlugin.js

@@ -0,0 +1,77 @@
+import path from 'path';
+import { cli } from '../cli/cli.js';
+import { ReferenceIssue } from '../issues/ReferenceIssue.js';
+import { Plugin } from './Plugin.js';
+
+/** @typedef {import('../assets/HtmlPage.js').HtmlPage} HtmlPage */
+/** @typedef {import('../../types/main.js').Reference} Reference */
+/** @typedef {import('../../types/main.js').CheckContext} CheckContext */
+/** @typedef {import('../../types/main.js').AddToQueueHelpers} AddToQueueHelpers */
+
+/**
+ * @param {string} url
+ */
+function getDomain(url) {
+  try {
+    const { hostname } = new URL(url);
+    return hostname;
+  } catch {
+    return '';
+  }
+}
+
+export class ExternalReferencesPlugin extends Plugin {
+  domainStats = new Map();
+
+  constructor(options = {}) {
+    super({
+      title: 'External',
+      checkLabel: 'links',
+      ...options,
+    });
+  }
+
+  /**
+   * @param {CheckContext} context
+   */
+  async check(context) {
+    const { item, report, getAsset } = context;
+    const reference = /** @type {Reference} */ (item);
+    const asset = getAsset(reference.url);
+
+    if (!(await asset.exists())) {
+      const { page } = reference;
+      const relPath = path.relative(process.cwd(), page.localPath);
+      const filePath = `./${relPath}:${reference.start}`;
+      const message = `<${reference.tag} ${reference.attribute}="${reference.value}">`;
+      report(new ReferenceIssue({ page, filePath, message }));
+    }
+  }
+
+  /**
+   * @param {HtmlPage} page
+   * @param {AddToQueueHelpers} helpers
+   * @returns {Promise<Reference[]>}
+   */
+  async addToQueue(page, helpers) {
+    const { isLocalUrl } = helpers;
+    const checkItems = [];
+    for (const reference of page.references) {
+      if (reference.url.startsWith('http') && !isLocalUrl(reference.url)) {
+        checkItems.push(reference);
+        const domain = getDomain(reference.url);
+        this.domainStats.set(domain, (this.domainStats.get(domain) || 0) + 1);
+      }
+    }
+    return checkItems;
+  }
+
+  render() {
+    const top3 = [...this.domainStats.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3);
+    const top3Str = top3.map(([domain, count], i) => `${i + 1}. ${domain} (${count})`).join(', ');
+    return cli`
+      ${super.render()}
+      - Top Domains: ${top3Str}
+    `;
+  }
+}

packages/check-website/src/plugins/HasCanonicalPlugin.js

@@ -0,0 +1,60 @@
+import path from 'path';
+import { PageIssue } from '../issues/PageIssue.js';
+import { Plugin } from './Plugin.js';
+
+/** @typedef {import('../assets/HtmlPage.js').HtmlPage} HtmlPage */
+/** @typedef {import('../../types/main.js').CheckContext} CheckContext */
+/** @typedef {import('../../types/main.js').Reference} Reference */
+/** @typedef {import('../../types/main.js').ParseElement} ParseElement */
+
+export class HasCanonicalPlugin extends Plugin {
+  /**
+   * Contains the unique absolute page URL as declared by the
+   * `<link rel="canonical" href="...">` element (if any) for every page.
+   *
+   * @type {Map<HtmlPage, URL>}
+   */
+  canonicalUrls = new Map();
+
+  /**
+   * @param {ParseElement} element
+   * @param {HtmlPage} page
+   */
+  onParseElement = (element, page) => {
+    if (!this.canonicalUrls.has(page) && element.tagName === 'LINK') {
+      if (element.getAttribute('rel') === 'canonical') {
+        const href = element.getAttribute('href');
+        if (href) {
+          this.canonicalUrls.set(page, new URL(href));
+        }
+      }
+    }
+  };
+
+  constructor(options = {}) {
+    super({
+      title: 'Canonical',
+      checkLabel: 'pages',
+      ...options,
+    });
+  }
+
+  /**
+   * @param {CheckContext} context
+   */
+  async check(context) {
+    const page = /** @type {HtmlPage} */ (context.item);
+    if (!this.canonicalUrls.has(page)) {
+      context.report(
+        new PageIssue({
+          title: 'Missing canonical',
+          message: 'The page is missing a <link rel="canonical" href="...">',
+          page,
+          filePath: './' + path.relative(process.cwd(), page.localPath),
+          icon: '🦄',
+        }),
+      );
+    }
+    return;
+  }
+}

packages/check-website/src/plugins/LocalReferencesPlugin.js

@@ -0,0 +1,68 @@
+import normalizeUrl from 'normalize-url';
+import path from 'path';
+import { HtmlPage } from '../assets/HtmlPage.js';
+import { ReferenceIssue } from '../issues/ReferenceIssue.js';
+import { Plugin } from './Plugin.js';
+
+/** @typedef {import('../../types/main.js').Reference} Reference */
+/** @typedef {import('../../types/main.js').CheckContext} CheckContext */
+/** @typedef {import('../../types/main.js').AddToQueueHelpers} AddToQueueHelpers */
+/** @typedef {import('../../types/main.js').PluginInterface} PluginInterface */
+
+/**
+ * @implement {PluginInterface}
+ */
+export class LocalReferencesPlugin extends Plugin {
+  constructor(options = {}) {
+    super({
+      title: 'Local',
+      checkLabel: 'links',
+      ...options,
+    });
+  }
+
+  /**
+   * @param {HtmlPage} page
+   * @param {AddToQueueHelpers} helpers
+   * @returns {Promise<Reference[]>}
+   */
+  async addToQueue(page, helpers) {
+    const { isLocalUrl } = helpers;
+    const checkItems = [];
+    for (const reference of page.references) {
+      if (isLocalUrl(reference.url)) {
+        checkItems.push(reference);
+      }
+    }
+    return checkItems;
+  }
+
+  /**
+   * @param {CheckContext} context
+   */
+  async check(context) {
+    const { item, report, getAsset } = context;
+    const reference = /** @type {Reference} */ (item);
+    const targetAsset = getAsset(reference.url);
+
+    const { page } = reference;
+    const relPath = path.relative(process.cwd(), page.localPath);
+    const filePath = `./${relPath}:${reference.start}`;
+    const message = `<${reference.tag} ${reference.attribute}="${reference.value}">`;
+
+    if (await targetAsset.exists()) {
+      const urlFocusHash = normalizeUrl(reference.url, { stripTextFragment: true }); // removes :~:text=
+      const hash = urlFocusHash.includes('#') ? urlFocusHash.split('#')[1] : '';
+      if (
+        hash &&
+        targetAsset instanceof HtmlPage &&
+        (await targetAsset.parse()) &&
+        !targetAsset.hasHash(hash)
+      ) {
+        report(new ReferenceIssue({ page, filePath, message, icon: '#️⃣' }));
+      }
+    } else {
+      report(new ReferenceIssue({ page, filePath, message }));
+    }
+  }
+}

packages/check-website/src/plugins/Plugin.js

@@ -0,0 +1,215 @@
+import { gray, green, red } from 'colorette';
+import { EventEmitter } from 'events';
+import { ASSET_STATUS } from '../assets/Asset.js';
+import { HtmlPage } from '../assets/HtmlPage.js';
+import { renderProgressBar } from '../cli/renderProgressBar.js';
+import { Queue } from '../helpers/Queue.js';
+
+/** @typedef {import('../assets/Asset.js').Asset} Asset */
+/** @typedef {import('../CheckWebsiteCli.js').CheckWebsiteCli} CheckWebsiteCli */
+
+/** @typedef {import('../../types/main.js').Reference} Reference */
+/** @typedef {import('../../types/main.js').CheckContext} CheckContext */
+/** @typedef {import('../../types/main.js').AddToQueueHelpers} AddToQueueHelpers */
+/** @typedef {import('../../types/main.js').PluginInterface} PluginInterface */
+
+export class Plugin {
+  /** @type {import('../issues/IssueManager.js').IssueManager | undefined} */
+  issueManager;
+
+  /** @type {import('../assets/AssetManager.js').AssetManager | undefined} */
+  assetManager;
+
+  /** @type {CheckWebsiteCli | undefined} */
+  cli;
+
+  _passed = 0;
+  _failed = 0;
+  _skipped = 0;
+
+  /**
+   * @type {[number, number] | undefined}
+   */
+  _performanceStart;
+
+  /**
+   * @type {Map<string, unknown>}
+   */
+  _checkItems = new Map();
+
+  _queue = new Queue();
+
+  _processedPages = new Set();
+
+  /**
+   * @readonly
+   */
+  events = new EventEmitter();
+
+  /**
+   * @param {Asset} asset
+   */
+  async onNewParsedAsset(asset) {
+    if (asset instanceof HtmlPage) {
+      asset.events.on('status-changed', async () => {
+        if (asset.status >= ASSET_STATUS.parsed) {
+          if (!this._processedPages.has(asset)) {
+            this._processedPages.add(asset);
+            /** @type {AddToQueueHelpers} */
+            const helpers = {
+              isLocalUrl: url => this.isLocalUrl(url),
+            };
+            const newQueueItems = await this.addToQueue(asset, helpers);
+            newQueueItems.forEach(_item => {
+              this._queue.add(async () => {
+                const item = /** @type {Reference | HtmlPage} */ (_item);
+
+                let skip = false;
+                if (item.url) {
+                  const url = item.url instanceof URL ? item.url.href : item.url;
+                  const targetAsset = this.assetManager?.getAsset(url);
+                  if (this.isLocalUrl(url)) {
+                    if (targetAsset instanceof HtmlPage) {
+                      targetAsset.parse(); // no await but we request the parse => e.g. we crawl
+                    }
+                  }
+                  if (targetAsset?.options.skip) {
+                    skip = true;
+                  }
+                }
+
+                if (skip === false) {
+                  let hadIssues = false;
+                  /** @type {CheckContext} */
+                  const context = {
+                    report: issue => {
+                      hadIssues = true;
+                      this.issueManager?.add(issue);
+                    },
+                    item,
+                    getAsset: url => {
+                      if (!this.assetManager) {
+                        throw Error('Asset manager not available');
+                      }
+                      return this.assetManager.getAsset(url);
+                    },
+                    isLocalUrl: url => this.isLocalUrl(url),
+                  };
+                  await /** @type {PluginInterface} */ (/** @type {unknown} */ (this)).check(
+                    context,
+                  );
+                  if (hadIssues) {
+                    this._failed += 1;
+                  } else {
+                    this._passed += 1;
+                  }
+                } else {
+                  this._skipped += 1;
+                }
+
+                this.events.emit('progress');
+              });
+            });
+          }
+        }
+      });
+    }
+  }
+
+  /**
+   * @param {HtmlPage} page
+   * @param {AddToQueueHelpers} helpers
+   * @returns {Promise<unknown[]>}
+   */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async addToQueue(page, helpers) {
+    return [page];
+  }
+
+  /**
+   * @param {Partial<{}>} options
+   */
+  constructor(options = {}) {
+    this.options = {
+      title: 'Plugin',
+      checkLabel: 'pages',
+      ...options,
+    };
+
+    if (this.options.title.length > 10) {
+      throw new Error(`Plugin title should be max 10 characters. Given "${this.options.title}"`);
+    }
+
+    this._queue.on('idle', () => {
+      this.events.emit('idle');
+    });
+  }
+
+  get isIdle() {
+    return this._queue.isIdle;
+  }
+
+  /**
+   * @param {CheckWebsiteCli} cli
+   */
+  setup(cli) {
+    this._performanceStart = process.hrtime();
+    this.cli = cli;
+  }
+
+  getTotal() {
+    return this._queue.getTotal();
+  }
+
+  getDuration() {
+    return this._queue.getDuration();
+  }
+
+  getDone() {
+    return this._queue.getDone();
+  }
+
+  getPassed() {
+    return this._passed;
+  }
+
+  getFailed() {
+    return this._failed;
+  }
+
+  getSkipped() {
+    return this._skipped;
+  }
+
+  /**
+   * @param {string} url
+   * @returns {boolean}
+   */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  isLocalUrl(url) {
+    return true;
+  }
+
+  render() {
+    const checkLabel = this.options.checkLabel;
+    const doneNr = this.getDone();
+    const passed = this.getPassed();
+    const failed = this.getFailed();
+    const skipped = this.getSkipped();
+    const total = this.getTotal();
+
+    const title = `${this.options.title}:`.padEnd(11);
+    const progress = renderProgressBar(doneNr, 0, total);
+
+    const minNumberLength = `${total}`.length;
+    const done = `${doneNr}`.padStart(minNumberLength);
+
+    const passedTxt = passed > 0 ? `${green(`${passed} passed`)}` : '0 passed';
+    const failedTxt = failed > 0 ? `, ${red(`${failed} failed`)}` : '';
+    const skippedTxt = skipped > 0 ? `, ${gray(`${skipped} skipped`)}` : '';
+    const resultTxt = `${passedTxt}${failedTxt}${skippedTxt}`;
+    const duration = this.getDuration();
+
+    return `${title} ${progress} ${done}/${total} ${checkLabel} | 🕑 ${duration}s | ${resultTxt}`;
+  }
+}

packages/check-website/test-node/00-Asset.test.js

@@ -0,0 +1,56 @@
+import { expect } from 'chai';
+import { HtmlPage, AssetManager } from '../src/index.js';
+
+const testOptions = {
+  originUrl: 'https://example.com/',
+  originPath: new URL('.', import.meta.url).pathname,
+};
+
+describe('Asset', () => {
+  it('01: add local file via file url exists', async () => {
+    const assets = new AssetManager(testOptions);
+    const asset = assets.addExistingFile(
+      new URL('fixtures/01-AssetManager/file.txt', import.meta.url),
+    );
+    expect(await asset.exists()).to.be.true;
+  });
+
+  it('01b: local file via http url exists if added before', async () => {
+    const assets = new AssetManager(testOptions);
+    assets.addExistingFile(new URL('fixtures/01-AssetManager/file.txt', import.meta.url));
+    const asset = assets.addUrl(new URL('https://example.com/fixtures/01-AssetManager/file.txt'));
+    expect(await asset.exists()).to.be.true;
+  });
+
+  it('01c: local file missing', async () => {
+    const assets = new AssetManager(testOptions);
+    const asset = assets.addUrl(
+      new URL('https://example.com/fixtures/01-AssetManager/missing.txt'),
+    );
+    expect(await asset.exists()).to.be.false;
+  });
+
+  it('01d: local html page exists', async () => {
+    const assets = new AssetManager(testOptions);
+    const page = assets.addExistingFile(
+      new URL('fixtures/01-AssetManager/page.html', import.meta.url),
+    );
+    expect(page).to.be.an.instanceOf(HtmlPage);
+    expect(await page.exists()).to.be.true;
+  });
+
+  it('02: external file exists', async () => {
+    const assets = new AssetManager(testOptions);
+    const asset = assets.addUrl(new URL('https://rocket.modern-web.dev/favicon.ico'));
+    expect(await asset.exists()).to.be.true;
+  });
+
+  it('03: adds assets while parsing local pages', async () => {
+    const assets = new AssetManager(testOptions);
+    const page = /** @type {HtmlPage} */ (
+      assets.addExistingFile(new URL('fixtures/01-AssetManager/page.html', import.meta.url))
+    );
+    await page.parse();
+    expect(assets.size).to.equal(2);
+  });
+});

packages/check-website/test-node/01-HtmlPage.test.js

@@ -0,0 +1,452 @@
+import chai from 'chai';
+import path from 'path';
+import { Readable } from 'stream';
+import { readFile } from 'fs/promises';
+
+import { HtmlPage, ASSET_STATUS, AssetManager } from 'check-website';
+import { fileURLToPath } from 'url';
+
+class FakeReadable extends Readable {
+  _read() {
+    // noop
+  }
+}
+
+class MockedFetch {
+  constructor({ content = '' } = {}) {
+    this.stream = new FakeReadable();
+    this._content = content;
+  }
+
+  get fetch() {
+    setTimeout(async () => {
+      this.stream.push(this._content);
+      this.stream.emit('end');
+    }, 1);
+    return () => Promise.resolve({ ok: true, body: this.stream });
+  }
+
+  /**
+   * @param {any} chunk
+   */
+  push(chunk) {
+    return this.stream.push(chunk);
+  }
+}
+
+const { expect } = chai;
+
+const currentDir = path.dirname(new URL(import.meta.url).pathname);
+
+/**
+ * @param {HtmlPage} page
+ */
+function cleanup(page) {
+  const keep = {};
+  keep.hashes = page.hashes;
+  keep.redirectTargetUrl = page.redirectTargetUrl;
+  keep.references = page.references.map(ref => ({
+    url: ref.url,
+    attribute: ref.attribute,
+    tag: ref.tag,
+    value: ref.value,
+  }));
+  const optKeep = {};
+  optKeep.localPath = page.options.localPath
+    ? `abs::${path.relative(currentDir, page.options.localPath)}`
+    : page.options.localPath;
+  optKeep.localSourcePath = page.options.localSourcePath
+    ? `abs::${path.relative(currentDir, page.options.localSourcePath)}`
+    : page.options.localSourcePath;
+  keep.options = optKeep;
+  keep.url = page.url;
+  return keep;
+}
+
+/**
+ * @param {import('../types/main.js').HtmlPageOptions} options
+ */
+function withTestOptions(options) {
+  return {
+    originUrl: 'https://example.com/',
+    originPath: new URL('.', import.meta.url).pathname,
+    assetManager: new AssetManager(),
+    ...options,
+  };
+}
+
+describe('HtmlPage', () => {
+  it('01: hashes', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/01-hashes.html'),
+      withTestOptions({
+        localPath: fileURLToPath(new URL('fixtures/01-HtmlPage/01-hashes.html', import.meta.url)),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/01-hashes.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/01-hashes.html',
+        localSourcePath: '',
+      },
+      hashes: ['first', 'second', 'third'],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          value: '#',
+          url: 'https://example.com/fixtures/01-HtmlPage/01-hashes.html#',
+        },
+      ],
+    });
+  });
+
+  it.skip('01a: fetch it as an external url', async () => {
+    const mocked = new MockedFetch({
+      // @ts-ignore
+      content: await readFile(new URL('fixtures/01-HtmlPage/01-hashes.html', import.meta.url)),
+    });
+    const page = new HtmlPage(
+      new URL('https://is.mocked.com/'),
+      withTestOptions({
+        // @ts-ignore
+        fetch: mocked.fetch,
+      }),
+    );
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://is.mocked.com/'),
+      options: {
+        localPath: '',
+        localSourcePath: '',
+      },
+      hashes: ['first', 'second', 'third'],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          value: '#',
+          url: 'https://is.mocked.com/#',
+        },
+      ],
+    });
+  });
+
+  it('02: internal link', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/02-internal-link.html'),
+      withTestOptions({
+        localPath: fileURLToPath(
+          new URL('fixtures/01-HtmlPage/02-internal-link.html', import.meta.url),
+        ),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/02-internal-link.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/02-internal-link.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'https://example.com/index.html',
+          value: '/index.html',
+        },
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'https://example.com/fixtures/01-HtmlPage/index.html',
+          value: './index.html',
+        },
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'https://example.com/fixtures/01-HtmlPage/index.html#first-headline',
+          value: './index.html#first-headline',
+        },
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'https://example.com/fixtures/01-HtmlPage/index.html?data=in&query=params',
+          value: './index.html?data=in&query=params',
+        },
+      ],
+    });
+  });
+
+  it('03: images', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/03-images.html'),
+      withTestOptions({
+        localPath: fileURLToPath(new URL('fixtures/01-HtmlPage/03-images.html', import.meta.url)),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/03-images.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/03-images.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'src',
+          tag: 'img',
+          url: 'https://example.com/empty.png',
+          value: '/empty.png',
+        },
+        {
+          attribute: 'src',
+          tag: 'img',
+          url: 'https://example.com/fixtures/01-HtmlPage/empty.png',
+          value: './empty.png',
+        },
+        {
+          attribute: 'src',
+          tag: 'img',
+          url: 'https://example.com/fixtures/01-HtmlPage/empty.png?data=in&query=params',
+          value: './empty.png?data=in&query=params',
+        },
+      ],
+    });
+  });
+
+  it('04: picture with srcset', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/04-picture-srcset.html'),
+      withTestOptions({
+        localPath: fileURLToPath(
+          new URL('fixtures/01-HtmlPage/04-picture-srcset.html', import.meta.url),
+        ),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/04-picture-srcset.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/04-picture-srcset.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'src',
+          tag: 'img',
+          url: 'https://example.com/images/empty.png',
+          value: '/images/empty.png',
+        },
+        {
+          attribute: 'srcset',
+          tag: 'source',
+          url: 'https://example.com/images/empty-300.png',
+          value: '/images/empty-300.png 300w, /images/empty-600.png?data=in&query=params 600w',
+        },
+        {
+          attribute: 'srcset',
+          tag: 'source',
+          url: 'https://example.com/images/empty-600.png?data=in&query=params',
+          value: '/images/empty-300.png 300w, /images/empty-600.png?data=in&query=params 600w',
+        },
+        {
+          attribute: 'src',
+          tag: 'img',
+          url: 'https://example.com/images/missing.png',
+          value: '/images/missing.png',
+        },
+        {
+          attribute: 'srcset',
+          tag: 'source',
+          url: 'https://example.com/images/missing-300.png',
+          value: '/images/missing-300.png 300w, /images/missing-600.png 600w',
+        },
+        {
+          attribute: 'srcset',
+          tag: 'source',
+          url: 'https://example.com/images/missing-600.png',
+          value: '/images/missing-300.png 300w, /images/missing-600.png 600w',
+        },
+      ],
+    });
+  });
+
+  it('05: mailto', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/05-mailto.html'),
+      withTestOptions({
+        localPath: fileURLToPath(new URL('fixtures/01-HtmlPage/05-mailto.html', import.meta.url)),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/05-mailto.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/05-mailto.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'mailto:foo@bar.com',
+          value: 'mailto:foo@bar.com',
+        },
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'mailto:address@example.com',
+          value:
+            'mailto:address@example.com',
+        },
+      ],
+    });
+  });
+
+  it('06: not http schema', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/06-not-http-schema.html'),
+      withTestOptions({
+        localPath: fileURLToPath(
+          new URL('fixtures/01-HtmlPage/06-not-http-schema.html', import.meta.url),
+        ),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/06-not-http-schema.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/06-not-http-schema.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'sketch://add-library?url=https%3A%2F%2Fmyexample.com%2Fdesign%2Fui-kit.xml',
+          value: 'sketch://add-library?url=https%3A%2F%2Fmyexample.com%2Fdesign%2Fui-kit.xml',
+        },
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'vscode://file/c:/myProject/package.json:5:10',
+          value: 'vscode://file/c:/myProject/package.json:5:10',
+        },
+      ],
+    });
+  });
+
+  it('07: tel', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/07-tel.html'),
+      withTestOptions({
+        localPath: fileURLToPath(new URL('fixtures/01-HtmlPage/07-tel.html', import.meta.url)),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/07-tel.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/07-tel.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'tel:99999',
+          value: 'tel:99999',
+        },
+      ],
+    });
+  });
+
+  it('08: ignore about schema', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/08-ignore-about-schema.html'),
+      withTestOptions({
+        localPath: fileURLToPath(
+          new URL('fixtures/01-HtmlPage/08-ignore-about-schema.html', import.meta.url),
+        ),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/08-ignore-about-schema.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/08-ignore-about-schema.html',
+        localSourcePath: '',
+      },
+      hashes: [],
+      redirectTargetUrl: undefined,
+      references: [
+        {
+          attribute: 'href',
+          tag: 'a',
+          url: 'about:dino',
+          value: 'about:dino',
+        },
+      ],
+    });
+  });
+
+  it('09: html meta refresh', async () => {
+    const page = new HtmlPage(
+      new URL('https://example.com/fixtures/01-HtmlPage/09-html-meta-refresh.html'),
+      withTestOptions({
+        localPath: fileURLToPath(
+          new URL('fixtures/01-HtmlPage/09-html-meta-refresh.html', import.meta.url),
+        ),
+      }),
+    );
+    page.status = ASSET_STATUS.existsLocal;
+    await page.parse();
+
+    // the HtmlPage starting it is not added as we only create it for the test
+    // but the redirect it found does get added to the asset manager
+    expect(page.options.assetManager?.size).to.equal(1);
+
+    expect(cleanup(page)).to.deep.equal({
+      url: new URL('https://example.com/fixtures/01-HtmlPage/09-html-meta-refresh.html'),
+      options: {
+        localPath: 'abs::fixtures/01-HtmlPage/09-html-meta-refresh.html',
+        localSourcePath: '',
+      },
+      redirectTargetUrl: new URL('https://example.com/en/getting-started'),
+      hashes: [],
+      references: [],
+    });
+  });
+});

packages/check-website/test-node/02-gatherFiles.test.js

@@ -0,0 +1,68 @@
+import { expect } from 'chai';
+import path from 'path';
+import { gatherFiles } from '../src/helpers/gatherFiles.js';
+
+const currentDir = path.dirname(new URL(import.meta.url).pathname);
+
+/**
+ * @param {string[]} files
+ */
+function cleanupFiles(files) {
+  return files.map(file => (file ? `abs::${path.relative(currentDir, file)}` : file));
+}
+
+describe('gatherFiles', () => {
+  it('01: current dir', async () => {
+    const files = await gatherFiles(
+      new URL('fixtures/02-gatherFiles/01-current-dir/', import.meta.url),
+    );
+
+    expect(cleanupFiles(files)).to.deep.equal([
+      'abs::fixtures/02-gatherFiles/01-current-dir/a.html',
+      'abs::fixtures/02-gatherFiles/01-current-dir/b.js',
+      'abs::fixtures/02-gatherFiles/01-current-dir/c.html',
+    ]);
+  });
+
+  it('02: sub dir goes files first', async () => {
+    const files = await gatherFiles(
+      new URL('fixtures/02-gatherFiles/02-sub-dir/', import.meta.url),
+    );
+
+    expect(cleanupFiles(files)).to.deep.equal([
+      'abs::fixtures/02-gatherFiles/02-sub-dir/a.html',
+      'abs::fixtures/02-gatherFiles/02-sub-dir/z.html',
+      'abs::fixtures/02-gatherFiles/02-sub-dir/sub/b.html',
+      'abs::fixtures/02-gatherFiles/02-sub-dir/sub/c.html',
+      'abs::fixtures/02-gatherFiles/02-sub-dir/sub/some.js',
+      'abs::fixtures/02-gatherFiles/02-sub-dir/sub2/d.html',
+    ]);
+  });
+
+  it('03: index.html file always goes first', async () => {
+    const files = await gatherFiles(
+      new URL('fixtures/02-gatherFiles/03-index-first/', import.meta.url),
+    );
+
+    expect(cleanupFiles(files)).to.deep.equal([
+      'abs::fixtures/02-gatherFiles/03-index-first/index.html',
+      'abs::fixtures/02-gatherFiles/03-index-first/about.html',
+    ]);
+  });
+
+  it('03b: index.html file always goes first with sub dirs', async () => {
+    const files = await gatherFiles(
+      new URL('fixtures/02-gatherFiles/03b-index-first-sub-dir/', import.meta.url),
+    );
+
+    expect(cleanupFiles(files)).to.deep.equal([
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/index.html',
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/about.html',
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub/index.html',
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub/a.html',
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub2/index.html',
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub2/b.html',
+      'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub2/z.html',
+    ]);
+  });
+});

packages/check-website/test-node/03-LocalReferencesPlugin.test.js

@@ -0,0 +1,105 @@
+import { expect } from 'chai';
+import { red, green } from 'colorette';
+
+import { LocalReferencesPlugin } from 'check-website';
+import { setupTestCli } from './test-helpers.js';
+
+function getOptions() {
+  return {
+    originUrl: 'https://example.com',
+    plugins: [new LocalReferencesPlugin()],
+  };
+}
+
+describe('LocalReferencesPlugin', () => {
+  it('01: finds a missing page', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/01-page-missing',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  it('01b: finds a variation of missing pages', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/01b-page-missing-variations',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('6 passed')}, ${red('4 failed')}`);
+  });
+
+  it('02: finds a missing hash', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/02-hash-missing',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  it('03: can identify full urls as internal', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/03-absolute-url-as-internal',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  it('04: automatically finds internal base url via canonical url', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/04-auto-finds-internal-base-url',
+      { plugins: [new LocalReferencesPlugin()] },
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('2 passed')}, ${red('1 failed')}`);
+  });
+
+  it('05: missing asset text file', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/05-asset-missing',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  it('06: starts crawling from the root index.html and ignores unlinked pages', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/06-crawls-from-root-index',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('2 failed')}`);
+  });
+
+  it('07: allow to mix urls with www and none www prefix', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/07-mix-www-and-none-www',
+      { plugins: [new LocalReferencesPlugin()] },
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('3 passed')}, ${red('1 failed')}`);
+  });
+
+  it('08: handle url in a case insensitive way', async () => {
+    const { capturedLogs, execute } = await setupTestCli(
+      'fixtures/03-LocalReferencesPlugin/08-handle-urls-case-insensitive',
+      { plugins: [new LocalReferencesPlugin()] },
+      { captureLogs: true },
+    );
+    await execute();
+    // tests today-I-learned vs today-i-learned
+    expect(capturedLogs.join('\n')).to.include(`${green('2 passed')}, ${red('1 failed')}`);
+  });
+});

packages/check-website/test-node/04-ExternalReferencesPlugin.test.js

@@ -0,0 +1,49 @@
+import { expect } from 'chai';
+import { red, green } from 'colorette';
+
+import { ExternalReferencesPlugin } from 'check-website';
+import { setupTestCli } from './test-helpers.js';
+
+function getOptions() {
+  return {
+    originUrl: 'https://example.com',
+    plugins: [new ExternalReferencesPlugin()],
+  };
+}
+
+describe('ExternalReferencePlugin', () => {
+  it('01: finds a missing page', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/04-ExternalReferencePlugin/01-page-missing',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  // it('02: finds a missing hash', async () => {
+  //   const { cli } = await setupTestCli('fixtures/04-ExternalReferencePlugin/02-hash-missing');
+  //   await cli.start();
+  // });
+
+  it('03: image service using a none http url', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/04-ExternalReferencePlugin/03-image-service-none-http',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  it('04: data urls are not interpreted as external', async () => {
+    const { execute, getLastDynamicLog } = await setupTestCli(
+      'fixtures/04-ExternalReferencePlugin/04-data-urls-are-not-external',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(getLastDynamicLog()).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+});

packages/check-website/test-node/05-HasCanonicalPlugin.test.js

@@ -0,0 +1,34 @@
+import { expect } from 'chai';
+import { red, green } from 'colorette';
+
+import { HasCanonicalPlugin } from 'check-website';
+import { setupTestCli } from './test-helpers.js';
+
+function getOptions() {
+  return {
+    originUrl: 'https://example.com',
+    plugins: [new HasCanonicalPlugin()],
+  };
+}
+
+describe('HasCanonicalPlugin', () => {
+  it('01: with a canonical', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/05-HasCanonicalPlugin/01-with-canonical',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}`);
+  });
+
+  it('02: without a canonical', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/05-HasCanonicalPlugin/02-without-canonical',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${red('1 failed')}`);
+  });
+});

packages/check-website/test-node/06-config.test.js

@@ -0,0 +1,45 @@
+import { expect } from 'chai';
+import { red, green, gray } from 'colorette';
+
+import { LocalReferencesPlugin, ExternalReferencesPlugin } from 'check-website';
+import { setupTestCli } from './test-helpers.js';
+
+function getOptions() {
+  return {
+    originUrl: 'https://example.com',
+    plugins: [new LocalReferencesPlugin()],
+  };
+}
+
+describe('Config', () => {
+  it('01: reads config file', async () => {
+    const { execute, capturedLogs, cli } = await setupTestCli(
+      'fixtures/06-config/01-change-origin-url/site',
+      { plugins: [new LocalReferencesPlugin()] },
+      { captureLogs: true },
+    );
+    await execute();
+    expect(cli.options.originUrl).to.equal('https://some-domain.com');
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
+  });
+
+  it('02: supports skips', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/06-config/02-skips/site',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${gray('1 skipped')}`);
+  });
+
+  it('03: supports skips for external', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/06-config/03-skips-external/site',
+      { originUrl: 'https://example.com', plugins: [new ExternalReferencesPlugin()] },
+      { captureLogs: true },
+    );
+    await execute();
+    expect(capturedLogs.join('\n')).to.include(`${gray('2 skipped')}`);
+  });
+});

packages/check-website/test-node/07-display.test.js

@@ -0,0 +1,32 @@
+import { expect } from 'chai';
+
+import { LocalReferencesPlugin, ExternalReferencesPlugin } from 'check-website';
+import { setupTestCli } from './test-helpers.js';
+
+function getOptions() {
+  return {
+    originUrl: 'https://example.com',
+    plugins: [new LocalReferencesPlugin(), new ExternalReferencesPlugin()],
+  };
+}
+
+describe('Display', () => {
+  it('01: duration is correct for multiple plugins', async () => {
+    const { execute, capturedLogs } = await setupTestCli(
+      'fixtures/07-display/01-duration',
+      getOptions(),
+      { captureLogs: true },
+    );
+    await execute();
+
+    const lastDynamic = capturedLogs
+      .join('\n')
+      .split('────────────────────────────────────────────────────────────────────────────────')
+      // @ts-ignore
+      .at(-1);
+    const matches = lastDynamic.match(/\| 1\/1 links \| 🕑 (.*)s \|/g);
+
+    expect(matches).to.have.lengthOf(2);
+    expect(matches[0]).to.not.equal(matches[1]);
+  });
+});

packages/check-website/test-node/fixtures/01-AssetManager/page.html

@@ -0,0 +1 @@
+<a href="./file.txt">File</a>

packages/check-website/test-node/fixtures/01-HtmlPage/01-hashes.html

@@ -0,0 +1,5 @@
+<h1 id="first">First</h1>
+<p>Lorem ipsum</p>
+<h2 id="second">Second</h2>
+<input name="first-name" />
+<a href="#" name="third">Third</a>

packages/check-website/test-node/fixtures/01-HtmlPage/02-internal-link.html

@@ -0,0 +1,4 @@
+<a href="/index.html">index</a>
+<a href="./index.html">index</a>
+<a href="./index.html#first-headline">index</a>
+<a href="./index.html?data=in&query=params">index</a>

packages/check-website/test-node/fixtures/01-HtmlPage/03-images.html

@@ -0,0 +1,3 @@
+<img src="/empty.png" alt="" />
+<img src="./empty.png" alt="" />
+<img src="./empty.png?data=in&query=params" alt="" />

packages/check-website/test-node/fixtures/01-HtmlPage/04-picture-srcset.html

@@ -0,0 +1,9 @@
+<picture>
+  <source srcset="/images/empty-300.png 300w, /images/empty-600.png?data=in&query=params 600w" type="image/jpeg" sizes="(min-width: 62.5em) 25vw, (min-width: 30.625em) 50vw, 100vw">
+  <img src="/images/empty.png" alt="Empty" width="300" height="225">
+</picture>
+
+<picture>
+  <source srcset="/images/missing-300.png 300w, /images/missing-600.png 600w" type="image/jpeg" sizes="(min-width: 62.5em) 25vw, (min-width: 30.625em) 50vw, 100vw">
+  <img src="/images/missing.png" alt="Empty" width="300" height="225">
+</picture>

packages/check-website/test-node/fixtures/01-HtmlPage/05-mailto.html

@@ -0,0 +1,3 @@
+<a href="mailto:foo@bar.com"></a>
+<!-- encoded mailto links -->
+<a href="&#109;&#97;&#105;&#108;&#116;&#111;&#58;&#97;&#100;&#100;&#114;&#101;&#115;&#115;&#64;&#101;&#120;&#97;&#109;&#112;&#108;&#101;&#46;&#99;&#111;&#109;"></a>