mirror of
https://github.com/modernweb-dev/rocket.git
synced 2026-03-23 00:41:19 +00:00
Compare commits
21 Commits
update
...
feat/check
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8d3492091d | ||
|
|
f5b344fe8e | ||
|
|
c8de46504e | ||
|
|
54c6e734d6 | ||
|
|
ec21b3f5c5 | ||
|
|
87966d1c7f | ||
|
|
30cd84811c | ||
|
|
b29209c512 | ||
|
|
3c29951213 | ||
|
|
35eb01101a | ||
|
|
404a152f63 | ||
|
|
f151cce24d | ||
|
|
c266bc0bd9 | ||
|
|
cb2d277830 | ||
|
|
8de14ed5ea | ||
|
|
dbb4d5b932 | ||
|
|
ea98aef699 | ||
|
|
f3f1feabda | ||
|
|
5037dbed2a | ||
|
|
d4e1508c70 | ||
|
|
57bcb84538 |
@@ -1,6 +0,0 @@
|
||||
---
|
||||
'@rocket/building-rollup': minor
|
||||
'@rocket/cli': minor
|
||||
---
|
||||
|
||||
Update to Rollup v3 + all plugins that require it
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
'@rocket/building-rollup': patch
|
||||
---
|
||||
|
||||
fix json5 security vulnerability
|
||||
@@ -14,7 +14,7 @@
|
||||
"@rocket/cli": "^0.20.0",
|
||||
"@rocket/engine": "^0.2.0",
|
||||
"@webcomponents/template-shadowroot": "^0.1.0",
|
||||
"lit": "^3.0.0"
|
||||
"lit": "^2.3.0"
|
||||
},
|
||||
"@rocket/template-name": "Hydration Starter",
|
||||
"imports": {
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"devDependencies": {
|
||||
"@rocket/cli": "^0.20.0",
|
||||
"@rocket/engine": "^0.2.0",
|
||||
"lit": "^3.0.0"
|
||||
"lit": "^2.3.0"
|
||||
},
|
||||
"@rocket/template-name": "Blog Starter"
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"devDependencies": {
|
||||
"@rocket/cli": "^0.20.0",
|
||||
"@rocket/engine": "^0.2.0",
|
||||
"lit": "^3.0.0"
|
||||
"lit": "^2.3.0"
|
||||
},
|
||||
"@rocket/template-name": "Minimal Starter"
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"@sanity/client": "^3.1.0",
|
||||
"@sanity/image-url": "^1.0.1",
|
||||
"dotenv": "^16.0.0",
|
||||
"lit": "^3.0.0"
|
||||
"lit": "^2.3.0"
|
||||
},
|
||||
"@rocket/template-name": "Sanity Minimal Starter"
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
"@rocket/components": "^0.2.0",
|
||||
"@rocket/engine": "^0.2.0",
|
||||
"@rocket/spark": "^0.2.0",
|
||||
"lit": "^3.0.0"
|
||||
"lit": "^2.3.0"
|
||||
},
|
||||
"@rocket/template-name": "Landing Page (@rocket/spark Theme)",
|
||||
"imports": {
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
"@rocket/engine": "^0.2.0",
|
||||
"@rocket/launch": "^0.21.0",
|
||||
"@rocket/search": "^0.7.0",
|
||||
"lit": "^3.0.0"
|
||||
"lit": "^2.3.0"
|
||||
},
|
||||
"@rocket/template-name": "Documentation Website (@rocket/launch Theme)",
|
||||
"imports": {
|
||||
|
||||
21328
package-lock.json
generated
21328
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
12
package.json
12
package.json
@@ -12,7 +12,6 @@
|
||||
"analyze:analyze": "node scripts/workspaces-scripts-bin.mjs analyze",
|
||||
"build": "npm run rocket:build",
|
||||
"build:site": "run-s analyze:* rocket:build",
|
||||
"build:start": "npm run rocket:build && cd _site && npx http-server -o -p 9000",
|
||||
"changeset": "changeset",
|
||||
"debug": "web-test-runner --watch --config web-test-runner-chrome.config.mjs",
|
||||
"debug:firefox": "web-test-runner --watch --config web-test-runner-firefox.config.mjs",
|
||||
@@ -33,7 +32,6 @@
|
||||
"search": "node packages/cli/src/cli.js search",
|
||||
"start": "NODE_DEBUG=engine:rendering node --trace-warnings packages/cli/src/cli.js start --open",
|
||||
"start:experimental": "NODE_DEBUG=engine:rendering node --no-warnings --experimental-loader ./packages/engine/src/litCssLoader.js packages/cli/src/cli.js start --open",
|
||||
"start:build": "cd _site && npx http-server -o -p 9000",
|
||||
"test": "npm run test:node && npm run test:web",
|
||||
"test:integration": "playwright test packages/*/test-node/*.spec.js --retries=3",
|
||||
"test:node": "npm run test:unit && npm run test:integration",
|
||||
@@ -50,10 +48,9 @@
|
||||
"@custom-elements-manifest/analyzer": "^0.4.12",
|
||||
"@open-wc/testing": "^3.1.2",
|
||||
"@playwright/test": "^1.18.1",
|
||||
"@rollup/plugin-commonjs": "^25.0.0",
|
||||
"@rollup/plugin-json": "^6.0.0",
|
||||
"@rollup/plugin-terser": "^0.4.0",
|
||||
"@rollup/plugin-typescript": "^11.1.0",
|
||||
"@rollup/plugin-commonjs": "^17.0.0",
|
||||
"@rollup/plugin-json": "^4.1.0",
|
||||
"@rollup/plugin-typescript": "^8.1.0",
|
||||
"@types/chai": "^4.2.14",
|
||||
"@types/fs-extra": "^9.0.6",
|
||||
"@types/mocha": "^9.0.0",
|
||||
@@ -85,7 +82,8 @@
|
||||
"puppeteer": "^13.0.0",
|
||||
"remark-emoji": "^2.1.0",
|
||||
"rimraf": "^3.0.2",
|
||||
"rollup": "^3.0.0",
|
||||
"rollup": "^2.36.1",
|
||||
"rollup-plugin-terser": "^7.0.2",
|
||||
"sinon": "^9.2.3",
|
||||
"ts-node": "^9.1.1",
|
||||
"typescript": "^4.8.4",
|
||||
|
||||
@@ -52,20 +52,20 @@
|
||||
"rollup"
|
||||
],
|
||||
"peerDependencies": {
|
||||
"rollup": "^3.0.0"
|
||||
"rollup": "^2.35.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@babel/core": "^7.20.12",
|
||||
"@babel/core": "^7.12.10",
|
||||
"@babel/preset-env": "^7.12.11",
|
||||
"@rollup/plugin-babel": "^6.0.0",
|
||||
"@rollup/plugin-babel": "^5.2.2",
|
||||
"@rollup/plugin-node-resolve": "^15.0.1",
|
||||
"@rollup/plugin-replace": "^5.0.1",
|
||||
"@rollup/plugin-terser": "^0.4.0",
|
||||
"@web/rollup-plugin-html": "^2.0.0",
|
||||
"@web/rollup-plugin-import-meta-assets": "^2.0.0",
|
||||
"@web/rollup-plugin-polyfills-loader": "^2.0.0",
|
||||
"@web/rollup-plugin-html": "^1.8.0",
|
||||
"@web/rollup-plugin-import-meta-assets": "^1.0.4",
|
||||
"@web/rollup-plugin-polyfills-loader": "^1.1.0",
|
||||
"browserslist": "^4.16.1",
|
||||
"plugins-manager": "^0.3.1",
|
||||
"rollup-plugin-terser": "^7.0.2",
|
||||
"workbox-broadcast-update": "^6.1.5",
|
||||
"workbox-cacheable-response": "^6.1.5",
|
||||
"workbox-expiration": "^6.1.5",
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import resolve from '@rollup/plugin-node-resolve';
|
||||
import terser from '@rollup/plugin-terser';
|
||||
import { babel } from '@rollup/plugin-babel';
|
||||
// @ts-ignore
|
||||
import { terser } from 'rollup-plugin-terser';
|
||||
import babelPkg from '@rollup/plugin-babel';
|
||||
|
||||
import { applyPlugins } from 'plugins-manager';
|
||||
|
||||
const { babel } = babelPkg;
|
||||
|
||||
/** @typedef {import('../types/main.js').BuildingRollupOptions} BuildingRollupOptions */
|
||||
|
||||
/**
|
||||
@@ -81,7 +84,6 @@ export function createBasicMetaConfig(userConfig = { output: {} }) {
|
||||
},
|
||||
},
|
||||
{
|
||||
// @ts-ignore
|
||||
plugin: terser,
|
||||
options: {},
|
||||
},
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import resolve from '@rollup/plugin-node-resolve';
|
||||
import terser from '@rollup/plugin-terser';
|
||||
import { babel } from '@rollup/plugin-babel';
|
||||
// @ts-ignore
|
||||
import { terser } from 'rollup-plugin-terser';
|
||||
import babelPkg from '@rollup/plugin-babel';
|
||||
import replace from '@rollup/plugin-replace';
|
||||
|
||||
import { applyPlugins } from 'plugins-manager';
|
||||
|
||||
const { babel } = babelPkg;
|
||||
|
||||
/** @typedef {import('../types/main.js').BuildingRollupOptions} BuildingRollupOptions */
|
||||
|
||||
/**
|
||||
@@ -83,7 +86,6 @@ export function createServiceWorkerMetaConfig(userConfig = { output: {} }) {
|
||||
},
|
||||
},
|
||||
{
|
||||
// @ts-ignore
|
||||
plugin: terser,
|
||||
options: {
|
||||
mangle: {
|
||||
|
||||
87
packages/check-website/CHANGELOG.md
Normal file
87
packages/check-website/CHANGELOG.md
Normal file
@@ -0,0 +1,87 @@
|
||||
# check-html-links
|
||||
|
||||
## 0.2.4
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 97d5fb2: Add external links validation via the flag `--validate-externals`.
|
||||
|
||||
You can/should provide an optional `--absolute-base-url` to handle urls starting with it as internal.
|
||||
|
||||
```bash
|
||||
# check external urls
|
||||
npx check-html-links _site --validate-externals
|
||||
|
||||
# check external urls but treat links like https://rocket.modern-web.dev/about/ as internal
|
||||
npx check-html-links _site --validate-externals --absolute-base-url https://rocket.modern-web.dev
|
||||
```
|
||||
|
||||
## 0.2.3
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 5043429: Ignore `<a href="tel:9999">` links
|
||||
- f08f926: Add missing `slash` dependency
|
||||
- a0e8edf: Ignore links containing not http schema urls like `sketch://`, `vscode://`, ...
|
||||
|
||||
```html
|
||||
<a href="sketch://add-library?url=https%3A%2F%2Fmyexample.com%2Fdesign%2Fui-kit.xml"></a>
|
||||
<a href="vscode://file/c:/myProject/package.json:5:10"></a>
|
||||
```
|
||||
|
||||
- 1949b1e: Ignore plain and html encoded mailto links
|
||||
|
||||
```html
|
||||
<!-- source -->
|
||||
<a href="mailto:address@example.com">contact</a>
|
||||
|
||||
<!-- html encoded -->
|
||||
<a
|
||||
href="mailto:address@example.com"
|
||||
>contact</a
|
||||
>
|
||||
```
|
||||
|
||||
## 0.2.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 66c2d78: fix: windows path issue
|
||||
|
||||
## 0.2.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- be0d0b3: fix: add missing main entry to the packages
|
||||
|
||||
## 0.2.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 750418b: Uses a class for the CLI and adding the following options:
|
||||
|
||||
- `--root-dir` the root directory to serve files from. Defaults to the current working directory
|
||||
- `--ignore-link-pattern` do not check links matching the pattern
|
||||
- `--continue-on-error` if present it will not exit with an error code - useful while writing or for temporary passing a ci
|
||||
|
||||
BREAKING CHANGE:
|
||||
|
||||
- Exists with an error code if a broken link is found
|
||||
|
||||
## 0.1.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- f343c50: When reading bigger files, especially bigger files with all content on one line it could mean a read chunk is in the middle of a character. This can lead to strange symbols in the resulting string. The `hightWaterMark` is now increased from the node default of 16KB to 256KB. Additionally, the `hightWaterMark` is now synced for reading and parsing.
|
||||
|
||||
## 0.1.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- eb74110: Add info about how many files and links will be checked
|
||||
|
||||
## 0.1.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- cd22231: Initial release
|
||||
28
packages/check-website/README.md
Normal file
28
packages/check-website/README.md
Normal file
@@ -0,0 +1,28 @@
|
||||
# Check Website
|
||||
|
||||
A fast checker for broken links/references in HTML.
|
||||
|
||||
## Installation
|
||||
|
||||
```shell
|
||||
npm i -D check-website
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
npx check-website
|
||||
```
|
||||
|
||||
For docs please see our homepage [https://rocket.modern-web.dev/tools/check-html-links/overview/](https://rocket.modern-web.dev/tools/check-html-links/overview/).
|
||||
|
||||
## Comparison
|
||||
|
||||
Checking the output of the [11ty-website](https://github.com/11ty/11ty-website) with 13 missing reference targets (used by 516 links) while checking 501 files. (on January 17, 2021)
|
||||
|
||||
| Tool | Lines printed | Times | Lang | Dependency Tree |
|
||||
| ---------------- | ------------- | ------ | ---- | --------------- |
|
||||
| check-html-links | 38 | ~2.5s | node | 19 |
|
||||
| link-checker | 3000+ | ~11s | node | 106 |
|
||||
| hyperlink | 68 | 4m 20s | node | 481 |
|
||||
| htmltest | 1000+ | ~0.7s | GO | - |
|
||||
3
packages/check-website/TODO.md
Normal file
3
packages/check-website/TODO.md
Normal file
@@ -0,0 +1,3 @@
|
||||
# Check HTML Links
|
||||
|
||||
- Support external entrypoints... e.g. user tried this `npx check-html-links@latest https://jasik.xyz`
|
||||
68
packages/check-website/package.json
Normal file
68
packages/check-website/package.json
Normal file
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"name": "check-website",
|
||||
"version": "0.0.1",
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"description": "A fast low dependency checker of html links/references",
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/modernweb-dev/rocket.git",
|
||||
"directory": "packages/check-website"
|
||||
},
|
||||
"author": "Modern Web <hello@modern-web.dev> (https://modern-web.dev/)",
|
||||
"homepage": "https://rocket.modern-web.dev/docs/tools/check-website/",
|
||||
"bin": {
|
||||
"check-website": "src/cli.js"
|
||||
},
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist-types/index.d.ts",
|
||||
"default": "./src/index.js"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"test": "mocha --timeout 5000 test-node/**/*.test.js test-node/*.test.js",
|
||||
"test:watch": "onchange 'src/**/*.js' 'test-node/**/*.js' -- npm test",
|
||||
"types": "wireit"
|
||||
},
|
||||
"files": [
|
||||
"dist-types",
|
||||
"src"
|
||||
],
|
||||
"dependencies": {
|
||||
"cacheable-lookup": "^7.0.0",
|
||||
"colorette": "^2.0.16",
|
||||
"commander": "^9.0.0",
|
||||
"got": "^12.5.2",
|
||||
"mime-types": "^2.1.35",
|
||||
"minimatch": "^3.0.4",
|
||||
"node-fetch": "^3.0.0",
|
||||
"normalize-url": "^7.2.0",
|
||||
"p-queue": "^7.3.0",
|
||||
"sax-wasm": "^2.0.0",
|
||||
"slash": "^4.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/mime-types": "^2.1.1"
|
||||
},
|
||||
"wireit": {
|
||||
"types": {
|
||||
"command": "tsc --build --pretty",
|
||||
"dependencies": [
|
||||
"../plugins-manager:types"
|
||||
],
|
||||
"files": [
|
||||
"src/**/*.js",
|
||||
"test-node/**/*.js",
|
||||
"types",
|
||||
"tsconfig.json"
|
||||
],
|
||||
"output": [
|
||||
"dist-types/**"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
217
packages/check-website/src/CheckWebsiteCli.js
Normal file
217
packages/check-website/src/CheckWebsiteCli.js
Normal file
@@ -0,0 +1,217 @@
|
||||
/* eslint-disable @typescript-eslint/ban-ts-comment */
|
||||
|
||||
/** @typedef {import('../types/main.js').FullCheckWebsiteCliOptions} FullCheckWebsiteCliOptions */
|
||||
/** @typedef {import('../types/main.js').CheckWebsiteCliOptions} CheckWebsiteCliOptions */
|
||||
/** @typedef {import('./assets/Asset.js').Asset} Asset */
|
||||
|
||||
import { EventEmitter } from 'events';
|
||||
import path from 'path';
|
||||
import { Command } from 'commander';
|
||||
import { green } from 'colorette';
|
||||
import { gatherFiles } from './helpers/gatherFiles.js';
|
||||
import { renderProgressBar } from './cli/renderProgressBar.js';
|
||||
import { LocalReferencesPlugin } from './plugins/LocalReferencesPlugin.js';
|
||||
// import { HasCanonicalPlugin } from './plugins/HasCanonicalPlugin.js';
|
||||
import { ExternalReferencesPlugin } from './plugins/ExternalReferencesPlugin.js';
|
||||
import { AssetManager } from './assets/AssetManager.js';
|
||||
import { LitTerminal } from './cli/LitTerminal.js';
|
||||
import { cli } from './cli/cli.js';
|
||||
import { IssueManager } from './issues/IssueManager.js';
|
||||
import { hr } from './cli/helpers.js';
|
||||
import { existsSync } from 'fs';
|
||||
import { readFile } from 'fs/promises';
|
||||
import { pathToFileURL } from 'url';
|
||||
import { normalizeUrl } from './helpers/normalizeUrl.js';
|
||||
import { HtmlPage } from './assets/HtmlPage.js';
|
||||
|
||||
export class CheckWebsiteCli extends LitTerminal {
|
||||
/** @type {FullCheckWebsiteCliOptions} */
|
||||
options = {
|
||||
configFile: '',
|
||||
inputDir: process.cwd(),
|
||||
originUrl: '',
|
||||
assetManager: new AssetManager(),
|
||||
issueManager: new IssueManager(),
|
||||
plugins: [],
|
||||
};
|
||||
|
||||
/**
|
||||
* @readonly
|
||||
*/
|
||||
events = new EventEmitter();
|
||||
|
||||
constructor({ argv = process.argv } = {}) {
|
||||
super();
|
||||
this.argv = argv;
|
||||
|
||||
this.program = new Command();
|
||||
this.program.allowUnknownOption().option('-c, --config-file <path>', 'path to config file');
|
||||
this.program.parse(this.argv);
|
||||
|
||||
this.program.allowUnknownOption(false);
|
||||
|
||||
if (this.program.opts().configFile) {
|
||||
this.options.configFile = this.program.opts().configFile;
|
||||
}
|
||||
|
||||
this.program
|
||||
.option('-i, --input-dir <path>', 'path to where to search for source files')
|
||||
.action(async cliOptions => {
|
||||
this.setOptions(cliOptions);
|
||||
});
|
||||
|
||||
this.options.plugins = [
|
||||
//
|
||||
new LocalReferencesPlugin(),
|
||||
// new HasCanonicalPlugin(),
|
||||
new ExternalReferencesPlugin(),
|
||||
];
|
||||
|
||||
/** @param {string} msg */
|
||||
this.options.issueManager.logger = msg => this.logStatic(msg);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Partial<CheckWebsiteCliOptions>} newOptions
|
||||
*/
|
||||
setOptions(newOptions) {
|
||||
this.options = {
|
||||
...this.options,
|
||||
...newOptions,
|
||||
};
|
||||
}
|
||||
|
||||
async applyConfigFile() {
|
||||
if (this.options.configFile) {
|
||||
const configFilePath = path.resolve(this.options.configFile);
|
||||
const fileOptions = (await import(configFilePath)).default;
|
||||
this.setOptions(fileOptions);
|
||||
} else {
|
||||
// make sure all default settings are properly initialized
|
||||
this.setOptions({});
|
||||
}
|
||||
}
|
||||
|
||||
async execute() {
|
||||
super.execute();
|
||||
await this.applyConfigFile();
|
||||
// const inputDir = userInputDir ? path.resolve(userInputDir) : process.cwd();
|
||||
|
||||
let entrypoint = path.join(this.options.inputDir, 'index.html');
|
||||
if (!existsSync(entrypoint)) {
|
||||
console.log(`Entrypoint ${entrypoint} does not exist`);
|
||||
process.exit(1);
|
||||
}
|
||||
if (!this.options.originUrl) {
|
||||
const entryHtml = await readFile(entrypoint, 'utf-8');
|
||||
const canonicalUrl = findCanonicalUrl(entryHtml);
|
||||
if (canonicalUrl) {
|
||||
this.options.originUrl = canonicalUrl;
|
||||
} else {
|
||||
console.log(`No canonical url found in ${entrypoint}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.options.isLocalUrl) {
|
||||
/** @param {string} url */
|
||||
this.options.isLocalUrl = url => {
|
||||
const normalizedUrl = normalizeUrl(url);
|
||||
return normalizedUrl.startsWith(this.options.originUrl);
|
||||
};
|
||||
}
|
||||
|
||||
this.logStatic('🔎 Check Website');
|
||||
this.logStatic('');
|
||||
this.logStatic(`👀 Start crawling from ${green('index.html')}`);
|
||||
this.logStatic(`📄 Will follow all links within ${green(this.options.originUrl)}`);
|
||||
this.logStatic('');
|
||||
|
||||
const onParseElementCallbacks = this.options.plugins
|
||||
.map(plugin => plugin.onParseElement)
|
||||
.filter(Boolean);
|
||||
|
||||
this.options.assetManager = new AssetManager({
|
||||
originPath: this.options.inputDir,
|
||||
originUrl: this.options.originUrl,
|
||||
// TODO: fix type...
|
||||
// @ts-ignore
|
||||
onParseElementCallbacks,
|
||||
plugins: this.options.plugins,
|
||||
isLocalUrl: this.options.isLocalUrl,
|
||||
skips: this.options.skips,
|
||||
});
|
||||
const pluginsAndAssetManager = [...this.options.plugins, this.options.assetManager];
|
||||
|
||||
this.options.assetManager.events.on('idle', () => {
|
||||
if (pluginsAndAssetManager.every(p => p.isIdle)) {
|
||||
this.updateComplete.then(() => {
|
||||
this.events.emit('done');
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
for (const plugin of this.options.plugins) {
|
||||
plugin.assetManager = this.options.assetManager;
|
||||
plugin.issueManager = this.options.issueManager;
|
||||
plugin.isLocalUrl = this.options.isLocalUrl;
|
||||
await plugin.setup(this);
|
||||
plugin.events.on('progress', () => {
|
||||
this.requestUpdate();
|
||||
});
|
||||
plugin.events.on('idle', () => {
|
||||
if (pluginsAndAssetManager.every(p => p.isIdle)) {
|
||||
this.updateComplete.then(() => {
|
||||
this.events.emit('done');
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const files = await gatherFiles(this.options.inputDir);
|
||||
if (files.length === 0) {
|
||||
console.log('🧐 No files to check. Did you select the correct folder?');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
for (const [, file] of files.entries()) {
|
||||
const fileUrl = pathToFileURL(file);
|
||||
this.options.assetManager.addExistingFile(fileUrl);
|
||||
}
|
||||
|
||||
// start crawling at the main index.html
|
||||
const rootPage = this.options.assetManager.get(this.options.originUrl + '/index.html');
|
||||
if (rootPage && rootPage instanceof HtmlPage) {
|
||||
await rootPage.parse();
|
||||
}
|
||||
}
|
||||
|
||||
renderParsing() {
|
||||
const title = `Parsing:`.padEnd(11);
|
||||
const total = this.options.assetManager.parsingQueue.getTotal();
|
||||
const doneNr = this.options.assetManager.parsingQueue.getDone();
|
||||
const duration = this.options.assetManager.parsingQueue.getDuration();
|
||||
const progress = renderProgressBar(doneNr, 0, total);
|
||||
const minNumberLength = `${total}`.length;
|
||||
const done = `${doneNr}`.padStart(minNumberLength);
|
||||
return `${title} ${progress} ${done}/${total} files | 🕑 ${duration}s`;
|
||||
}
|
||||
|
||||
render() {
|
||||
return cli`
|
||||
\n${hr()}
|
||||
${this.renderParsing()}
|
||||
${this.options.plugins.map(plugin => plugin.render()).join('\n')}
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} html
|
||||
*/
|
||||
function findCanonicalUrl(html) {
|
||||
const matches = html.match(/<link\s*rel="canonical"\s*href="(.*)"/);
|
||||
if (matches) {
|
||||
return normalizeUrl(matches[1]);
|
||||
}
|
||||
}
|
||||
138
packages/check-website/src/assets/Asset.js
Normal file
138
packages/check-website/src/assets/Asset.js
Normal file
@@ -0,0 +1,138 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import fetch from 'node-fetch';
|
||||
import got, { RequestError } from 'got';
|
||||
import CacheableLookup from 'cacheable-lookup';
|
||||
|
||||
const dnsCache = new CacheableLookup();
|
||||
|
||||
/** @typedef {import('../../types/main.js').AssetStatus} AssetStatus */
|
||||
|
||||
export const ASSET_STATUS = /** @type {const} */ ({
|
||||
unknown: 0,
|
||||
|
||||
// 200+ exists
|
||||
exists: 200,
|
||||
existsLocal: 220,
|
||||
existsExternal: 230,
|
||||
|
||||
// 250+ parsed
|
||||
parsed: 250,
|
||||
parsedLocal: 260,
|
||||
parsedExternal: 270,
|
||||
|
||||
// 300+ redirect
|
||||
redirect: 300,
|
||||
|
||||
// 400+ error
|
||||
missing: 400,
|
||||
});
|
||||
|
||||
export class Asset {
|
||||
/**
|
||||
* The full page URL.
|
||||
*
|
||||
* Example: `https://docs.astro.build/en/getting-started/`
|
||||
* @type {URL}
|
||||
*/
|
||||
url;
|
||||
|
||||
/**
|
||||
* @type {AssetStatus}
|
||||
*/
|
||||
#status = ASSET_STATUS.unknown;
|
||||
|
||||
/**
|
||||
* @param {AssetStatus} status
|
||||
*/
|
||||
set status(status) {
|
||||
this.#status = status;
|
||||
this.events.emit('status-changed');
|
||||
}
|
||||
|
||||
get status() {
|
||||
return this.#status;
|
||||
}
|
||||
|
||||
/**
|
||||
* @readonly
|
||||
*/
|
||||
events = new EventEmitter();
|
||||
|
||||
localPath = '';
|
||||
|
||||
localSourcePath = '';
|
||||
|
||||
/** @type {import('../../types/main.js').FullAssetOptions} */
|
||||
options = {
|
||||
originUrl: '',
|
||||
originPath: '',
|
||||
localPath: '',
|
||||
localSourcePath: '',
|
||||
fetch,
|
||||
assetManager: undefined,
|
||||
isLocalUrl: url => url.startsWith(this.options.originUrl),
|
||||
skip: false,
|
||||
};
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {URL} url
|
||||
* @param {import('../../types/main.js').AssetOptions} options
|
||||
*/
|
||||
constructor(url, options) {
|
||||
this.options = { ...this.options, ...options };
|
||||
this.url = url;
|
||||
this.localPath = this.options.localPath;
|
||||
|
||||
if (this.url.protocol === 'file:') {
|
||||
throw new Error(`File protocol is not supported. Used by ${this.url.href}`);
|
||||
}
|
||||
}
|
||||
|
||||
exists() {
|
||||
return new Promise(resolve => {
|
||||
if (this.status > ASSET_STATUS.unknown && this.status < ASSET_STATUS.missing) {
|
||||
resolve(true);
|
||||
} else if (this.options.isLocalUrl(this.url.href)) {
|
||||
// Local assets need to be added upfront and are not dynamically discovered - potentially a feature for later
|
||||
resolve(false);
|
||||
} else {
|
||||
this.options.assetManager?.fetchQueue
|
||||
.add(async () => await this.executeExists())
|
||||
.finally(() => {
|
||||
resolve(this.status > ASSET_STATUS.unknown && this.status < ASSET_STATUS.missing);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async executeExists() {
|
||||
if (this.options.skip) {
|
||||
return;
|
||||
}
|
||||
// TODO: detect server redirects (301, 302, etc)?
|
||||
// const fetching = fetch(this.url.href, { method: 'HEAD', redirect: "error" });
|
||||
try {
|
||||
await got(this.url.href, {
|
||||
method: 'HEAD',
|
||||
retry: {
|
||||
limit: 1,
|
||||
calculateDelay: ({ computedValue }) => {
|
||||
return computedValue / 100;
|
||||
},
|
||||
},
|
||||
dnsCache,
|
||||
});
|
||||
this.status = ASSET_STATUS.existsExternal;
|
||||
} catch (err) {
|
||||
if (err instanceof RequestError) {
|
||||
this.status =
|
||||
/** @type {AssetStatus} */ (err?.response?.statusCode) || ASSET_STATUS.missing;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
isLocal() {
|
||||
return this.status === ASSET_STATUS.existsLocal || this.status === ASSET_STATUS.parsedLocal;
|
||||
}
|
||||
}
|
||||
220
packages/check-website/src/assets/AssetManager.js
Normal file
220
packages/check-website/src/assets/AssetManager.js
Normal file
@@ -0,0 +1,220 @@
|
||||
import fetch from 'node-fetch';
|
||||
import path from 'path';
|
||||
import { fileURLToPath, pathToFileURL } from 'url';
|
||||
import mime from 'mime-types';
|
||||
|
||||
import { Asset, ASSET_STATUS } from './Asset.js';
|
||||
import { HtmlPage } from './HtmlPage.js';
|
||||
import { normalizeUrl, normalizeToLocalUrl } from '../helpers/normalizeUrl.js';
|
||||
import { decodeNumberHtmlEntities } from '../helpers/decodeNumberHtmlEntities.js';
|
||||
import { Queue } from '../helpers/Queue.js';
|
||||
import EventEmitter from 'events';
|
||||
import minimatch from 'minimatch';
|
||||
|
||||
/** @typedef {import('../plugins/Plugin.js').Plugin} Plugin */
|
||||
|
||||
const classMap = {
|
||||
Asset,
|
||||
HtmlPage,
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @returns {URL}
|
||||
*/
|
||||
function getUrl(url) {
|
||||
try {
|
||||
return new URL(url);
|
||||
} catch (e) {
|
||||
// handle html encoded mailto links like <a href="mailto:">
|
||||
const decoded = decodeNumberHtmlEntities(url);
|
||||
return new URL(decoded);
|
||||
}
|
||||
}
|
||||
|
||||
export class AssetManager {
|
||||
/** @type {Map<string, Asset | HtmlPage>} */
|
||||
assets = new Map();
|
||||
|
||||
/**
|
||||
* @readonly
|
||||
*/
|
||||
events = new EventEmitter();
|
||||
|
||||
/** Queue *************************/
|
||||
parsingQueue = new Queue({
|
||||
concurrency: 1,
|
||||
});
|
||||
|
||||
fetchQueue = new Queue({
|
||||
concurrency: 1,
|
||||
carryoverConcurrencyCount: true,
|
||||
interval: 500,
|
||||
intervalCap: 10,
|
||||
});
|
||||
|
||||
/** @type {import('../../types/main.js').FullAssetManagerOptions} */
|
||||
options = {
|
||||
originUrl: '',
|
||||
originPath: '',
|
||||
fetch,
|
||||
plugins: [],
|
||||
isLocalUrl: url => url.startsWith(this.options.originUrl),
|
||||
onParseElementCallbacks: [],
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {import('../../types/main.js').AssetManagerOptions} [options]
|
||||
*/
|
||||
constructor(options) {
|
||||
this.options = { ...this.options, ...options };
|
||||
|
||||
this.parsingQueue.on('idle', () => {
|
||||
this.events.emit('idle');
|
||||
});
|
||||
this.fetchQueue.on('idle', () => {
|
||||
this.events.emit('idle');
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} localPath
|
||||
* @returns {Asset | HtmlPage}
|
||||
*/
|
||||
addFile(localPath) {
|
||||
const url = pathToFileURL(localPath); // new URL('file://' + localPath);
|
||||
return this.addUrl(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* It does not check if the asset actually exits.
|
||||
* ONLY call it for asset urls you know exist.
|
||||
*
|
||||
* @param {URL} fileUrl
|
||||
* @return {Asset | HtmlPage}
|
||||
*/
|
||||
addExistingFile(fileUrl) {
|
||||
const filePath = fileURLToPath(fileUrl);
|
||||
const rel = path.relative(this.options.originPath, filePath);
|
||||
const url = new URL(rel, this.options.originUrl);
|
||||
const localPath = path.join(this.options.originPath, rel);
|
||||
if (this.has(url.href)) {
|
||||
const found = this.get(url.href);
|
||||
if (found) {
|
||||
return found;
|
||||
}
|
||||
}
|
||||
|
||||
const mimeType = mime.lookup(fileUrl.pathname);
|
||||
const skip = this.options.skips && this.options.skips.some(skip => minimatch(url.href, skip));
|
||||
|
||||
/** @type {keyof classMap} */
|
||||
let typeClass = 'Asset';
|
||||
if (mimeType === 'text/html') {
|
||||
typeClass = 'HtmlPage';
|
||||
}
|
||||
const asset = new classMap[typeClass](url, {
|
||||
assetManager: this,
|
||||
localPath,
|
||||
originPath: this.options.originPath,
|
||||
originUrl: this.options.originUrl,
|
||||
onParseElementCallbacks: this.options.onParseElementCallbacks,
|
||||
skip,
|
||||
});
|
||||
asset.status = ASSET_STATUS.existsLocal;
|
||||
this.assets.set(this.normalizeUrl(url.href), asset);
|
||||
|
||||
for (const plugin of this.options.plugins) {
|
||||
plugin.onNewParsedAsset(asset);
|
||||
}
|
||||
|
||||
return asset;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {URL | string} url
|
||||
* @param {{ mimeType?: string }} options
|
||||
* @returns {Asset | HtmlPage}
|
||||
*/
|
||||
addUrl(url, { mimeType = '' } = {}) {
|
||||
const useUrl = typeof url === 'string' ? getUrl(url) : url;
|
||||
if (this.has(useUrl.href)) {
|
||||
return /** @type {Asset | HtmlPage} */ (this.get(useUrl.href));
|
||||
}
|
||||
|
||||
const skip =
|
||||
this.options.skips && this.options.skips.some(skip => minimatch(useUrl.href, skip));
|
||||
|
||||
/** @type {keyof classMap} */
|
||||
let typeClass = 'Asset';
|
||||
if (mimeType === 'text/html') {
|
||||
typeClass = 'HtmlPage';
|
||||
}
|
||||
const asset = new classMap[typeClass](useUrl, {
|
||||
assetManager: this,
|
||||
originPath: this.options.originPath,
|
||||
originUrl: this.options.originUrl,
|
||||
onParseElementCallbacks: this.options.onParseElementCallbacks,
|
||||
skip,
|
||||
});
|
||||
|
||||
this.assets.set(this.normalizeUrl(useUrl.href), asset);
|
||||
|
||||
for (const plugin of this.options.plugins) {
|
||||
plugin.onNewParsedAsset(asset);
|
||||
}
|
||||
|
||||
return asset;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @returns {string}
|
||||
*/
|
||||
normalizeUrl(url) {
|
||||
if (this.options.isLocalUrl(url)) {
|
||||
return normalizeToLocalUrl(url);
|
||||
}
|
||||
return normalizeUrl(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @returns
|
||||
*/
|
||||
get(url) {
|
||||
return this.assets.get(this.normalizeUrl(url));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @returns {boolean}
|
||||
*/
|
||||
has(url) {
|
||||
return this.assets.has(this.normalizeUrl(url));
|
||||
}
|
||||
|
||||
get size() {
|
||||
return this.assets.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @returns {Asset | HtmlPage}
|
||||
*/
|
||||
getAsset(url) {
|
||||
let asset = this.get(url);
|
||||
if (!asset) {
|
||||
asset = this.addUrl(new URL(url));
|
||||
}
|
||||
return asset;
|
||||
}
|
||||
|
||||
getAll() {
|
||||
return Array.from(this.assets.values());
|
||||
}
|
||||
|
||||
get isIdle() {
|
||||
return this.fetchQueue.isIdle && this.parsingQueue.isIdle;
|
||||
}
|
||||
}
|
||||
246
packages/check-website/src/assets/HtmlPage.js
Normal file
246
packages/check-website/src/assets/HtmlPage.js
Normal file
@@ -0,0 +1,246 @@
|
||||
import fs from 'fs';
|
||||
|
||||
import { Asset, ASSET_STATUS } from './Asset.js';
|
||||
|
||||
import {
|
||||
getAttributeInfo,
|
||||
getLinksFromSrcSet,
|
||||
resolveToFullPageUrl,
|
||||
} from '../helpers/sax-helpers.js';
|
||||
import { parser, SaxEventType, streamOptions } from '../helpers/sax-parser.js';
|
||||
|
||||
/** @typedef {import('sax-wasm').Tag} Tag */
|
||||
/** @typedef {import('../../types/main.js').Reference} Reference */
|
||||
|
||||
export class HtmlPage extends Asset {
|
||||
/**
|
||||
* @type {Reference[]}
|
||||
*/
|
||||
references = [];
|
||||
|
||||
/**
|
||||
* A list of hashes that can be used as URL fragments to jump to specific parts of the page.
|
||||
* @type {string[]}
|
||||
*/
|
||||
hashes = [];
|
||||
|
||||
/**
|
||||
* The target URL of a `<meta http-equiv="refresh" content="...">` element
|
||||
* contained on the page (if any).
|
||||
* @type {URL | undefined}
|
||||
*/
|
||||
redirectTargetUrl;
|
||||
|
||||
referenceSources = [
|
||||
{ tagName: 'img', attribute: 'src' },
|
||||
{ tagName: 'img', attribute: 'srcset' },
|
||||
{ tagName: 'source', attribute: 'src' },
|
||||
{ tagName: 'source', attribute: 'srcset' },
|
||||
{ tagName: 'a', attribute: 'href' },
|
||||
{ tagName: 'link', attribute: 'href' },
|
||||
{ tagName: 'script', attribute: 'src' },
|
||||
];
|
||||
|
||||
/** @type {import('../../types/main.js').FullHtmlPageOptions} */
|
||||
options = {
|
||||
...this.options,
|
||||
onParseElementCallbacks: [],
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {URL} url
|
||||
* @param {import('../../types/main.js').HtmlPageOptions} options
|
||||
*/
|
||||
constructor(url, options) {
|
||||
super(url, {
|
||||
onParseElementCallbacks: [],
|
||||
...options,
|
||||
});
|
||||
this.options = { ...this.options, ...options };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} hash
|
||||
* @returns {boolean}
|
||||
*/
|
||||
hasHash(hash) {
|
||||
const checkHash = hash.startsWith('#') ? hash.substring(1) : hash;
|
||||
return this.hashes.includes(checkHash);
|
||||
}
|
||||
|
||||
async parse() {
|
||||
if (!(await this.exists())) {
|
||||
return;
|
||||
}
|
||||
|
||||
return await this._parse();
|
||||
}
|
||||
|
||||
#parsing = false;
|
||||
#parsingPromise = Promise.resolve(false);
|
||||
|
||||
/**
|
||||
*
|
||||
* @protected
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
_parse() {
|
||||
if (this.#parsing) {
|
||||
return this.#parsingPromise;
|
||||
}
|
||||
|
||||
this.#parsing = true;
|
||||
this.#parsingPromise = new Promise(resolve => {
|
||||
if (this.status >= ASSET_STATUS.parsed && this.status < ASSET_STATUS.missing) {
|
||||
this.#parsing = false;
|
||||
resolve(true);
|
||||
} else {
|
||||
if (!this.options.assetManager) {
|
||||
throw new Error('You need to pass an assetManager to the options');
|
||||
}
|
||||
this.options.assetManager?.parsingQueue
|
||||
.add(async () => await this.executeParse())
|
||||
.then(() => {
|
||||
this.#parsing = false;
|
||||
resolve(true);
|
||||
});
|
||||
}
|
||||
});
|
||||
return this.#parsingPromise;
|
||||
}
|
||||
|
||||
async executeParse() {
|
||||
if (this.options.skip) {
|
||||
return;
|
||||
}
|
||||
if (!(await this.exists())) {
|
||||
return;
|
||||
}
|
||||
await this._executeParse();
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @protected
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
_executeParse() {
|
||||
parser.eventHandler = (ev, _data) => {
|
||||
if (ev === SaxEventType.CloseTag) {
|
||||
const data = /** @type {Tag} */ (/** @type {any} */ (_data));
|
||||
|
||||
const searchTags = this.referenceSources.map(({ tagName }) => tagName);
|
||||
if (searchTags.includes(data.name)) {
|
||||
const possibleAttributes = this.referenceSources
|
||||
.map(({ attribute, tagName }) => (tagName === data.name ? attribute : undefined))
|
||||
.filter(Boolean);
|
||||
for (const possibleAttributeName of possibleAttributes) {
|
||||
if (possibleAttributeName) {
|
||||
const attribute = getAttributeInfo(data, possibleAttributeName);
|
||||
if (attribute) {
|
||||
const { value, start, end, name } = attribute;
|
||||
/** @type {Reference} */
|
||||
const entry = {
|
||||
start,
|
||||
end,
|
||||
value,
|
||||
url: resolveToFullPageUrl(this.url.href, value),
|
||||
page: this,
|
||||
attribute: name,
|
||||
tag: data.name,
|
||||
};
|
||||
if (name === 'srcset') {
|
||||
const links = getLinksFromSrcSet(value, this.url.href, entry);
|
||||
this.references.push(...links);
|
||||
} else {
|
||||
this.references.push(entry);
|
||||
if (this.status === ASSET_STATUS.existsLocal) {
|
||||
// only add "sub" assets for local files
|
||||
this.options.assetManager?.addUrl(entry.url);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const idData = getAttributeInfo(data, 'id');
|
||||
if (idData) {
|
||||
this.hashes.push(idData.value);
|
||||
}
|
||||
|
||||
if (data.name === 'a') {
|
||||
const nameData = getAttributeInfo(data, 'name');
|
||||
if (nameData) {
|
||||
this.hashes.push(nameData.value);
|
||||
}
|
||||
}
|
||||
|
||||
/** @type {import('../../types/main.js').ParseElement} */
|
||||
const element = {
|
||||
tagName: data.name.toUpperCase(),
|
||||
getAttribute: name => getAttributeInfo(data, name)?.value,
|
||||
};
|
||||
this.options.onParseElementCallbacks.forEach(cb => cb(element, this));
|
||||
|
||||
// Check if the page redirects somewhere else using meta refresh
|
||||
if (data.name === 'meta') {
|
||||
const httpEquivData = getAttributeInfo(data, 'http-equiv');
|
||||
if (httpEquivData && httpEquivData.value.toLowerCase() === 'refresh') {
|
||||
const metaRefreshContent = getAttributeInfo(data, 'content')?.value;
|
||||
const metaRefreshMatches = metaRefreshContent?.match(
|
||||
/^([0-9]+)\s*;\s*url\s*=\s*(.+)$/i,
|
||||
);
|
||||
this.redirectTargetUrl = metaRefreshMatches
|
||||
? new URL(metaRefreshMatches[2], this.url.href)
|
||||
: undefined;
|
||||
|
||||
if (this.status === ASSET_STATUS.existsLocal && this.redirectTargetUrl) {
|
||||
// only add "sub" assets for local files
|
||||
this.options.assetManager?.addUrl(this.redirectTargetUrl.href, {
|
||||
mimeType: 'text/html',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
if (this.status === ASSET_STATUS.existsLocal) {
|
||||
if (!this.options.localPath) {
|
||||
throw new Error(`Missing local path on the asset ${this.url.href}`);
|
||||
}
|
||||
// Read from FileSystem
|
||||
const readable = fs.createReadStream(this.options.localPath, streamOptions);
|
||||
readable.on('data', chunk => {
|
||||
// @ts-expect-error
|
||||
parser.write(chunk);
|
||||
});
|
||||
readable.on('end', () => {
|
||||
parser.end();
|
||||
this.status = ASSET_STATUS.parsedLocal;
|
||||
resolve();
|
||||
});
|
||||
}
|
||||
|
||||
if (this.status === ASSET_STATUS.existsExternal) {
|
||||
// Fetch from the network
|
||||
this.options.fetch(this.url.href).then(async response => {
|
||||
if (!response.ok || !response.body) {
|
||||
reject('Error in response');
|
||||
return;
|
||||
}
|
||||
response.body.on('data', chunk => {
|
||||
parser.write(chunk);
|
||||
});
|
||||
response.body.on('end', () => {
|
||||
parser.end();
|
||||
this.status = ASSET_STATUS.parsedExternal;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
29
packages/check-website/src/cli.js
Executable file
29
packages/check-website/src/cli.js
Executable file
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { existsSync } from 'fs';
|
||||
import path from 'path';
|
||||
import { CheckWebsiteCli } from './CheckWebsiteCli.js';
|
||||
|
||||
const cli = new CheckWebsiteCli();
|
||||
|
||||
const cwd = process.cwd();
|
||||
const configFiles = [
|
||||
path.join('config', 'check-website.config.js'),
|
||||
path.join('config', 'check-website.config.mjs'),
|
||||
'check-website.config.js',
|
||||
'check-website.config.mjs',
|
||||
path.join('..', 'config', 'check-website.config.js'),
|
||||
path.join('..', 'config', 'check-website.config.mjs'),
|
||||
path.join('..', 'check-website.config.js'),
|
||||
path.join('..', 'check-website.config.mjs'),
|
||||
];
|
||||
|
||||
for (const configFile of configFiles) {
|
||||
const configFilePath = path.join(cwd, configFile);
|
||||
if (existsSync(configFilePath)) {
|
||||
cli.options.configFile = configFilePath;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await cli.execute();
|
||||
383
packages/check-website/src/cli/LitTerminal.js
Normal file
383
packages/check-website/src/cli/LitTerminal.js
Normal file
@@ -0,0 +1,383 @@
|
||||
/* eslint-disable @typescript-eslint/ban-ts-comment */
|
||||
import { hideCursor } from './helpers.js';
|
||||
|
||||
const terminal = process.stderr;
|
||||
|
||||
export class LitTerminal {
|
||||
dynamicRender = true;
|
||||
|
||||
/**
|
||||
* They dynamic cli view.
|
||||
* It can only render as many lines as the terminal height which is usually around 24-40.
|
||||
* If you write more dynamic lines, an error will be thrown.
|
||||
* To display more information you can use logStatic() to render lines that will not by dynamic.
|
||||
*
|
||||
* Why?
|
||||
* If you write more lines than the terminal height then scrolling will happen and at this point it is
|
||||
* no longer accessible by the terminal itself.
|
||||
* Scrolling is a feature of terminal simulators, not the terminal itself.
|
||||
* This means that once content got scrolled out of the terminal, it is no longer adjustable.
|
||||
*
|
||||
* @example
|
||||
* render() {
|
||||
* return cli`
|
||||
* Counter: ${this.counter}
|
||||
* `;
|
||||
* }
|
||||
*
|
||||
* @returns {string}
|
||||
*/
|
||||
render() {
|
||||
return '';
|
||||
}
|
||||
|
||||
constructor() {
|
||||
/** @type {typeof LitTerminal} */ (this.constructor).__finalize();
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a static string that is rendered above the dynamic view.
|
||||
*
|
||||
* Use it to display information like a list of errors or completed tasks.
|
||||
* This content will not be cleared when the dynamic view is rendered and will always be reachable via scrolling.
|
||||
*
|
||||
* @param {string} message
|
||||
*/
|
||||
logStatic(message) {
|
||||
this.#staticLogs.push(message);
|
||||
this.requestUpdate();
|
||||
}
|
||||
|
||||
/**
|
||||
* Request a rerender of the dynamic view.
|
||||
* You can call this as often as you want and it will only rerender once per LitTerminal.renderInterval.
|
||||
* Typically you don't need to call this manually as it is called automatically when you update a property or use logStatic().
|
||||
* You can use it to create your own getters/setters
|
||||
* @example
|
||||
* _counter = 0;
|
||||
* set counter(counter) {
|
||||
* this._counter = counter;
|
||||
* this.requestUpdate();
|
||||
* }
|
||||
* get counter() {
|
||||
* return this._counter;
|
||||
* }s
|
||||
*/
|
||||
requestUpdate() {
|
||||
if (this.#updateRequested === false) {
|
||||
this.#updateRequested = true;
|
||||
setTimeout(() => {
|
||||
this._render();
|
||||
this.#updateRequested = false;
|
||||
}, /** @type {typeof LitTerminal} */ (this.constructor).renderInterval);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This actually
|
||||
*/
|
||||
execute() {
|
||||
hideCursor();
|
||||
}
|
||||
|
||||
//
|
||||
// End of public API
|
||||
//
|
||||
|
||||
#lastRender = '';
|
||||
|
||||
/** @type {string[]} */
|
||||
#staticLogs = [];
|
||||
|
||||
#updateRequested = false;
|
||||
|
||||
/**
|
||||
* How often (in ms) the dynamic view should be rerendered [defaults to 100]
|
||||
*/
|
||||
static renderInterval = 100;
|
||||
|
||||
/**
|
||||
* This writes the result of render() & logStatic() to the terminal.
|
||||
* It compares the last render with the current render and only clears and writes the changed lines.
|
||||
*
|
||||
* It throws an error you render() tries to write more lines than the terminal height.
|
||||
*/
|
||||
_render() {
|
||||
if (this.dynamicRender === false) {
|
||||
for (const staticLog of this.#staticLogs) {
|
||||
console.log(staticLog);
|
||||
}
|
||||
this.#staticLogs = [];
|
||||
return;
|
||||
}
|
||||
|
||||
const render = this.render();
|
||||
const renderLines = render.split('\n');
|
||||
|
||||
const windowSize = terminal.getWindowSize();
|
||||
if (renderLines.length > windowSize[1]) {
|
||||
throw new Error(
|
||||
`You rendered ${renderLines.length} lines while the terminal height is only ${windowSize[1]}. For non dynamic parts use logStatic()`,
|
||||
);
|
||||
}
|
||||
|
||||
if (render !== this.#lastRender || this.#staticLogs.length) {
|
||||
const lastRenderLines = this.#lastRender.split('\n');
|
||||
if (lastRenderLines.length > 0) {
|
||||
terminal.moveCursor(0, lastRenderLines.length * -1);
|
||||
terminal.cursorTo(0);
|
||||
}
|
||||
|
||||
const staticLength = this.#staticLogs.length;
|
||||
if (staticLength) {
|
||||
for (const staticLog of this.#staticLogs) {
|
||||
terminal.clearLine(0);
|
||||
console.log(staticLog);
|
||||
}
|
||||
this.#staticLogs = [];
|
||||
}
|
||||
|
||||
for (const [index, line] of renderLines.entries()) {
|
||||
if (line.length !== lastRenderLines[index - staticLength]?.length) {
|
||||
terminal.clearLine(0);
|
||||
}
|
||||
console.log(line);
|
||||
}
|
||||
terminal.clearScreenDown();
|
||||
|
||||
this.#lastRender = render;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// ***********************************************************************************************
|
||||
// Below if inspired by ReactiveElement
|
||||
// https://github.com/lit/lit/blob/main/packages/reactive-element/src/reactive-element.ts
|
||||
//
|
||||
|
||||
static finalized = false;
|
||||
static properties = {};
|
||||
|
||||
/**
|
||||
* Creates property accessors for registered properties, sets up element
|
||||
* styling, and ensures any superclasses are also finalized. Returns true if
|
||||
* the element was finalized.
|
||||
* @nocollapse
|
||||
*/
|
||||
static __finalize() {
|
||||
if (this.finalized) {
|
||||
return false;
|
||||
}
|
||||
this.finalized = true;
|
||||
// // finalize any superclasses
|
||||
// const superCtor = Object.getPrototypeOf(this);
|
||||
// superCtor.finalize();
|
||||
|
||||
if (this.properties) {
|
||||
const props = this.properties;
|
||||
// support symbols in properties (IE11 does not support this)
|
||||
const propKeys = [
|
||||
...Object.getOwnPropertyNames(props),
|
||||
...Object.getOwnPropertySymbols(props),
|
||||
];
|
||||
for (const p of propKeys) {
|
||||
// @ts-ignore
|
||||
this.createProperty(p, props[p]);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a property accessor on the element prototype if one does not exist
|
||||
* and stores a {@linkcode PropertyDeclaration} for the property with the
|
||||
* given options. The property setter calls the property's `hasChanged`
|
||||
* property option or uses a strict identity check to determine whether or not
|
||||
* to request an update.
|
||||
*
|
||||
* This method may be overridden to customize properties; however,
|
||||
* when doing so, it's important to call `super.createProperty` to ensure
|
||||
* the property is setup correctly. This method calls
|
||||
* `getPropertyDescriptor` internally to get a descriptor to install.
|
||||
* To customize what properties do when they are get or set, override
|
||||
* `getPropertyDescriptor`. To customize the options for a property,
|
||||
* implement `createProperty` like this:
|
||||
*
|
||||
* ```ts
|
||||
* static createProperty(name, options) {
|
||||
* options = Object.assign(options, {myOption: true});
|
||||
* super.createProperty(name, options);
|
||||
* }
|
||||
* ```
|
||||
*
|
||||
* @nocollapse
|
||||
* @category properties
|
||||
* @param {PropertyKey} name Name of property
|
||||
* @param {import('../../types/LitTerminal.js').PropertyDeclaration} options Property declaration
|
||||
*/
|
||||
static createProperty(name, options) {
|
||||
// Note, since this can be called by the `@property` decorator which
|
||||
// is called before `finalize`, we ensure finalization has been kicked off.
|
||||
this.__finalize();
|
||||
|
||||
this.elementProperties.set(name, options);
|
||||
|
||||
// Do not generate an accessor if the prototype already has one, since
|
||||
// it would be lost otherwise and that would never be the user's intention;
|
||||
// Instead, we expect users to call `requestUpdate` themselves from
|
||||
// user-defined accessors. Note that if the super has an accessor we will
|
||||
// still overwrite it
|
||||
// eslint-disable-next-line no-prototype-builtins
|
||||
if (!options.noAccessor && !this.prototype.hasOwnProperty(name)) {
|
||||
const key = typeof name === 'symbol' ? Symbol() : `__${name}`;
|
||||
const descriptor = this.getPropertyDescriptor(name, key, options);
|
||||
if (descriptor !== undefined) {
|
||||
Object.defineProperty(this.prototype, name, descriptor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a property descriptor to be defined on the given named property.
|
||||
* If no descriptor is returned, the property will not become an accessor.
|
||||
* For example,
|
||||
*
|
||||
* ```ts
|
||||
* class MyElement extends LitElement {
|
||||
* static getPropertyDescriptor(name, key, options) {
|
||||
* const defaultDescriptor =
|
||||
* super.getPropertyDescriptor(name, key, options);
|
||||
* const setter = defaultDescriptor.set;
|
||||
* return {
|
||||
* get: defaultDescriptor.get,
|
||||
* set(value) {
|
||||
* setter.call(this, value);
|
||||
* // custom action.
|
||||
* },
|
||||
* configurable: true,
|
||||
* enumerable: true
|
||||
* }
|
||||
* }
|
||||
* }
|
||||
* ```
|
||||
*
|
||||
* @nocollapse
|
||||
* @category properties
|
||||
* @param {PropertyKey} name
|
||||
* @param {string | symbol} key
|
||||
* @param {import('../../types/LitTerminal.js').PropertyDeclaration} options
|
||||
* @returns {PropertyDescriptor | undefined}
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
static getPropertyDescriptor(name, key, options) {
|
||||
return {
|
||||
/**
|
||||
* @this {LitTerminal}
|
||||
*/
|
||||
get() {
|
||||
// @ts-ignore
|
||||
return this[key];
|
||||
// return (this as {[key: string]: unknown})[key as string];
|
||||
},
|
||||
/**
|
||||
* @param {unknown} value
|
||||
* @this {LitTerminal}
|
||||
*/
|
||||
set(value) {
|
||||
// const oldValue = this[name];
|
||||
// @ts-ignore
|
||||
this[key] = value;
|
||||
this.requestUpdate();
|
||||
// this.requestUpdate(name, oldValue, options);
|
||||
},
|
||||
configurable: true,
|
||||
enumerable: true,
|
||||
};
|
||||
}
|
||||
|
||||
static elementProperties = new Map();
|
||||
|
||||
/**
|
||||
* Returns the property options associated with the given property.
|
||||
* These options are defined with a `PropertyDeclaration` via the `properties`
|
||||
* object or the `@property` decorator and are registered in
|
||||
* `createProperty(...)`.
|
||||
*
|
||||
* Note, this method should be considered "final" and not overridden. To
|
||||
* customize the options for a given property, override
|
||||
* {@linkcode createProperty}.
|
||||
*
|
||||
* @nocollapse
|
||||
* @final
|
||||
* @category properties
|
||||
* @param {string | symbol} name
|
||||
*/
|
||||
static getPropertyOptions(name) {
|
||||
return this.elementProperties.get(name) || defaultPropertyDeclaration;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a Promise that resolves when the element has completed updating.
|
||||
* The Promise value is a boolean that is `true` if the element completed the
|
||||
* update without triggering another update. The Promise result is `false` if
|
||||
* a property was set inside `updated()`. If the Promise is rejected, an
|
||||
* exception was thrown during the update.
|
||||
*
|
||||
* To await additional asynchronous work, override the `getUpdateComplete`
|
||||
* method. For example, it is sometimes useful to await a rendered element
|
||||
* before fulfilling this Promise. To do this, first await
|
||||
* `super.getUpdateComplete()`, then any subsequent state.
|
||||
*
|
||||
* @category updates
|
||||
* @returns {Promise<boolean>} A promise of a boolean that resolves to true if the update completed
|
||||
* without triggering another update.
|
||||
*/
|
||||
get updateComplete() {
|
||||
return new Promise(resolve => setTimeout(resolve, 100));
|
||||
// TODO: implement actual waiting for a finished render
|
||||
// return this.getUpdateComplete();
|
||||
}
|
||||
|
||||
// /**
|
||||
// * Override point for the `updateComplete` promise.
|
||||
// *
|
||||
// * It is not safe to override the `updateComplete` getter directly due to a
|
||||
// * limitation in TypeScript which means it is not possible to call a
|
||||
// * superclass getter (e.g. `super.updateComplete.then(...)`) when the target
|
||||
// * language is ES5 (https://github.com/microsoft/TypeScript/issues/338).
|
||||
// * This method should be overridden instead. For example:
|
||||
// *
|
||||
// * ```ts
|
||||
// * class MyElement extends LitElement {
|
||||
// * override async getUpdateComplete() {
|
||||
// * const result = await super.getUpdateComplete();
|
||||
// * await this._myChild.updateComplete;
|
||||
// * return result;
|
||||
// * }
|
||||
// * }
|
||||
// * ```
|
||||
// *
|
||||
// * @returns {Promise<boolean>} A promise of a boolean that resolves to true if the update completed
|
||||
// * without triggering another update.
|
||||
// * @category updates
|
||||
// */
|
||||
// getUpdateComplete() {
|
||||
// return this.__updatePromise;
|
||||
// }
|
||||
}
|
||||
|
||||
const defaultPropertyDeclaration = {
|
||||
type: String,
|
||||
hasChanged: notEqual,
|
||||
};
|
||||
|
||||
/**
|
||||
* Change function that returns true if `value` is different from `oldValue`.
|
||||
* This method is used as the default for a property's `hasChanged` function.
|
||||
* @param {unknown} value The new value
|
||||
* @param {unknown} old The old value
|
||||
*/
|
||||
export function notEqual(value, old) {
|
||||
// This ensures (old==NaN, value==NaN) always returns false
|
||||
return old !== value && (old === old || value === value);
|
||||
}
|
||||
72
packages/check-website/src/cli/cli.js
Normal file
72
packages/check-website/src/cli/cli.js
Normal file
@@ -0,0 +1,72 @@
|
||||
// TODO: support nested lists with new lines like: ${items.map(item => cli`[ ] ${item}\n`)}
|
||||
// will most likely require a stateful "cli tagged template literal" that passes on the current indent level
|
||||
|
||||
/**
|
||||
* Tagged template literal that
|
||||
* - dedents the string
|
||||
* - removes empty first/last lines
|
||||
* - joins arrays
|
||||
*
|
||||
* @example
|
||||
* const str = cli`
|
||||
* Welcome ${name}!
|
||||
* List: ${items.map(item => `[ ] ${item} `)}
|
||||
* `;
|
||||
* # becomes
|
||||
* Welcome John!
|
||||
* List: [ ] a [ ] b [ ] c
|
||||
*
|
||||
* @param {TemplateStringsArray} strings
|
||||
* @param {...any} values
|
||||
* @returns
|
||||
*/
|
||||
export function cli(strings, ...values) {
|
||||
const useStrings = typeof strings === 'string' ? [strings] : strings.raw;
|
||||
let result = '';
|
||||
|
||||
for (var i = 0; i < useStrings.length; i++) {
|
||||
let currentString = useStrings[i];
|
||||
currentString
|
||||
// join lines when there is a suppressed newline
|
||||
.replace(/\\\n[ \t]*/g, '')
|
||||
// handle escaped backticks
|
||||
.replace(/\\`/g, '`');
|
||||
result += currentString;
|
||||
|
||||
if (i < values.length) {
|
||||
const value = Array.isArray(values[i]) ? values[i].join('') : values[i];
|
||||
result += value;
|
||||
}
|
||||
}
|
||||
|
||||
// now strip indentation
|
||||
const lines = result.split('\n');
|
||||
let minIndent = -1;
|
||||
for (const line of lines) {
|
||||
const match = line.match(/^(\s+)\S+/);
|
||||
if (match) {
|
||||
const indent = match[1].length;
|
||||
if (minIndent === -1) {
|
||||
// this is the first indented line
|
||||
minIndent = indent;
|
||||
} else {
|
||||
minIndent = Math.min(minIndent, indent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let finalResult = '';
|
||||
if (minIndent !== -1) {
|
||||
for (const line of lines) {
|
||||
finalResult += line[0] === ' ' ? line.slice(minIndent) : line;
|
||||
finalResult += '\n';
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
finalResult
|
||||
.trim()
|
||||
// handle escaped newlines at the end to ensure they don't get stripped too
|
||||
.replace(/\\n/g, '\n')
|
||||
);
|
||||
}
|
||||
11
packages/check-website/src/cli/helpers.js
Normal file
11
packages/check-website/src/cli/helpers.js
Normal file
@@ -0,0 +1,11 @@
|
||||
export function hr(length = process.stderr.getWindowSize()[0]) {
|
||||
return '─'.repeat(length);
|
||||
}
|
||||
|
||||
export function hideCursor() {
|
||||
process.stderr.write('\u001B[?25l');
|
||||
}
|
||||
|
||||
export function showCursor() {
|
||||
process.stderr.write('\u001B[?25h');
|
||||
}
|
||||
40
packages/check-website/src/cli/renderProgressBar.js
Normal file
40
packages/check-website/src/cli/renderProgressBar.js
Normal file
@@ -0,0 +1,40 @@
|
||||
import { gray, white } from 'colorette';
|
||||
|
||||
const PROGRESS_BLOCKS = [' ', '▏', '▎', '▍', '▌', '▋', '▊', '▉', '█'];
|
||||
const PROGRESS_WIDTH = 30;
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {number} value
|
||||
* @param {number} total
|
||||
* @returns
|
||||
*/
|
||||
function createProgressBlocks(value, total) {
|
||||
if (value >= total) {
|
||||
return PROGRESS_BLOCKS[8].repeat(PROGRESS_WIDTH);
|
||||
}
|
||||
|
||||
const count = (PROGRESS_WIDTH * value) / total;
|
||||
const floored = Math.floor(count);
|
||||
const partialBlock =
|
||||
PROGRESS_BLOCKS[Math.floor((count - floored) * (PROGRESS_BLOCKS.length - 1))];
|
||||
return `${PROGRESS_BLOCKS[8].repeat(floored)}${partialBlock}${' '.repeat(
|
||||
PROGRESS_WIDTH - floored - 1,
|
||||
)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {number} finished
|
||||
* @param {number} active
|
||||
* @param {number} total
|
||||
* @returns
|
||||
*/
|
||||
export function renderProgressBar(finished, active, total) {
|
||||
const progressBlocks = createProgressBlocks(finished + active, total);
|
||||
const finishedBlockCount = Math.floor((PROGRESS_WIDTH * finished) / total);
|
||||
|
||||
const finishedBlocks = white(progressBlocks.slice(0, finishedBlockCount));
|
||||
const scheduledBlocks = gray(progressBlocks.slice(finishedBlockCount));
|
||||
return `|${finishedBlocks}${scheduledBlocks}|`;
|
||||
}
|
||||
50
packages/check-website/src/helpers/Queue.js
Normal file
50
packages/check-website/src/helpers/Queue.js
Normal file
@@ -0,0 +1,50 @@
|
||||
import { formatPerformance } from './formatPerformance.js';
|
||||
import PQueue from 'p-queue';
|
||||
|
||||
/** @typedef {import('p-queue').QueueAddOptions} QueueAddOptions */
|
||||
/** @typedef {import('p-queue').PriorityQueue} PriorityQueue */
|
||||
|
||||
export class Queue extends PQueue {
|
||||
#total = 0;
|
||||
|
||||
/** @type {[number, number] | undefined} */
|
||||
#durationStart;
|
||||
/** @type {[number, number] | undefined} */
|
||||
duration;
|
||||
|
||||
isIdle = true;
|
||||
|
||||
/**
|
||||
* @param {import('p-queue').Options<PriorityQueue, QueueAddOptions>} [options]
|
||||
*/
|
||||
constructor(options) {
|
||||
super(options);
|
||||
this.on('active', () => {
|
||||
if (!this.#durationStart) {
|
||||
this.#durationStart = process.hrtime();
|
||||
}
|
||||
this.isIdle = false;
|
||||
});
|
||||
this.on('completed', () => {
|
||||
this.duration = process.hrtime(this.#durationStart);
|
||||
});
|
||||
this.on('idle', () => {
|
||||
this.isIdle = true;
|
||||
});
|
||||
this.on('add', () => {
|
||||
this.#total += 1;
|
||||
});
|
||||
}
|
||||
|
||||
getDuration() {
|
||||
return this.duration ? formatPerformance(this.duration) : '0.00';
|
||||
}
|
||||
|
||||
getDone() {
|
||||
return this.#total - this.size - this.pending;
|
||||
}
|
||||
|
||||
getTotal() {
|
||||
return this.#total;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
/**
|
||||
* Converts number HTML entities to their corresponding characters.
|
||||
*
|
||||
* Example:
|
||||
* mailto: => mailto:
|
||||
*
|
||||
* @param {string} str
|
||||
* @returns {string}
|
||||
*/
|
||||
export function decodeNumberHtmlEntities(str) {
|
||||
return str.replace(/&#([0-9]{1,3});/gi, (match, numStr) => {
|
||||
const num = parseInt(numStr, 10); // read num as normal number
|
||||
return String.fromCharCode(num);
|
||||
});
|
||||
}
|
||||
7
packages/check-website/src/helpers/formatPerformance.js
Normal file
7
packages/check-website/src/helpers/formatPerformance.js
Normal file
@@ -0,0 +1,7 @@
|
||||
/**
|
||||
* @param {[number, number]} perf
|
||||
* @returns {string}
|
||||
*/
|
||||
export function formatPerformance(perf) {
|
||||
return (perf[0] + perf[1] / 1e9).toFixed(2);
|
||||
}
|
||||
45
packages/check-website/src/helpers/gatherFiles.js
Normal file
45
packages/check-website/src/helpers/gatherFiles.js
Normal file
@@ -0,0 +1,45 @@
|
||||
import { readdir } from 'fs/promises';
|
||||
import path from 'path';
|
||||
|
||||
/**
|
||||
* @param {string} fileName
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function isIndexFile(fileName) {
|
||||
return fileName === 'index.html';
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string | URL} inRootDir
|
||||
* @returns {Promise<string[]>}
|
||||
*/
|
||||
export async function gatherFiles(inRootDir) {
|
||||
const rootDir = inRootDir instanceof URL ? inRootDir.pathname : path.resolve(inRootDir);
|
||||
let files = [];
|
||||
|
||||
const entries = await readdir(rootDir, { withFileTypes: true });
|
||||
|
||||
// 1. handle possible index.html file
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory() && isIndexFile(entry.name)) {
|
||||
files.push(path.join(rootDir, entry.name));
|
||||
}
|
||||
}
|
||||
// 2. handle other html files
|
||||
for (const entry of entries) {
|
||||
const { name } = entry;
|
||||
if (entry.isFile() && !isIndexFile(name)) {
|
||||
const filePath = path.join(rootDir, name);
|
||||
files.push(filePath);
|
||||
}
|
||||
}
|
||||
// 3. handle sub directories
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) {
|
||||
const currentPath = path.join(rootDir, entry.name);
|
||||
files.push(...(await gatherFiles(currentPath)));
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
42
packages/check-website/src/helpers/normalizeUrl.js
Normal file
42
packages/check-website/src/helpers/normalizeUrl.js
Normal file
@@ -0,0 +1,42 @@
|
||||
import normalizeUrlDep from 'normalize-url';
|
||||
|
||||
const normalizeOptions = {
|
||||
stripAuthentication: true,
|
||||
stripHash: true,
|
||||
stripTextFragment: true,
|
||||
removeQueryParameters: false,
|
||||
// removeTrailingSlash: false,
|
||||
// removeSingleSlash: false,
|
||||
removeDirectoryIndex: true,
|
||||
// removeExplicitPort: true,
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @param {import('normalize-url').Options} options
|
||||
* @returns {string}
|
||||
*/
|
||||
export function normalizeUrl(url, options = {}) {
|
||||
if (url.startsWith('mailto:')) {
|
||||
return url.toLowerCase();
|
||||
}
|
||||
// = "mailto:" but html encoded)
|
||||
if (url.startsWith('mailto:')) {
|
||||
return url;
|
||||
}
|
||||
if (url.startsWith('tel:')) {
|
||||
return url.toLowerCase();
|
||||
}
|
||||
if (url.startsWith('about:')) {
|
||||
return url.toLowerCase();
|
||||
}
|
||||
return normalizeUrlDep(url, { ...normalizeOptions, ...options }).toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @returns {string}
|
||||
*/
|
||||
export function normalizeToLocalUrl(url) {
|
||||
return normalizeUrl(url, { removeQueryParameters: true });
|
||||
}
|
||||
66
packages/check-website/src/helpers/sax-helpers.js
Normal file
66
packages/check-website/src/helpers/sax-helpers.js
Normal file
@@ -0,0 +1,66 @@
|
||||
/** @typedef {import('sax-wasm').Tag} Tag */
|
||||
|
||||
/**
|
||||
* @param {Tag} data
|
||||
* @param {string} name
|
||||
*/
|
||||
export function getAttributeInfo(data, name) {
|
||||
if (data.attributes) {
|
||||
const { attributes } = data;
|
||||
const foundIndex = attributes.findIndex(entry => entry.name.value === name);
|
||||
if (foundIndex !== -1) {
|
||||
const entry = attributes[foundIndex].value;
|
||||
return {
|
||||
value: entry.value,
|
||||
start: `${entry.start.line + 1}:${entry.start.character}`,
|
||||
end: `${entry.end.line + 1}:${entry.end.character}`,
|
||||
name,
|
||||
};
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} pageUrl
|
||||
* @param {string} referenceUrl
|
||||
* @returns {string}
|
||||
*/
|
||||
export function resolveToFullPageUrl(pageUrl, referenceUrl) {
|
||||
// = "mailto:" but html encoded)
|
||||
if (referenceUrl.startsWith('mailto:')) {
|
||||
return referenceUrl;
|
||||
}
|
||||
if (referenceUrl.startsWith('about:')) {
|
||||
return referenceUrl;
|
||||
}
|
||||
const url = new URL(referenceUrl, pageUrl);
|
||||
return url.href;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} value Comma
|
||||
* @param {string} pageUrl
|
||||
* @param {import('../../types/main.js').Reference} entry
|
||||
* @returns
|
||||
*/
|
||||
export function getLinksFromSrcSet(value, pageUrl, entry) {
|
||||
const links = [];
|
||||
if (value.includes(',')) {
|
||||
const srcsets = value.split(',').map(el => el.trim());
|
||||
for (const srcset of srcsets) {
|
||||
if (srcset.includes(' ')) {
|
||||
const srcsetParts = srcset.split(' ');
|
||||
links.push({ ...entry, url: resolveToFullPageUrl(pageUrl, srcsetParts[0]) });
|
||||
} else {
|
||||
links.push({ ...entry, url: resolveToFullPageUrl(pageUrl, srcset) });
|
||||
}
|
||||
}
|
||||
} else if (value.includes(' ')) {
|
||||
const srcsetParts = value.split(' ');
|
||||
links.push({ ...entry, url: resolveToFullPageUrl(pageUrl, srcsetParts[0]) });
|
||||
} else {
|
||||
links.push(entry);
|
||||
}
|
||||
return links;
|
||||
}
|
||||
14
packages/check-website/src/helpers/sax-parser.js
Normal file
14
packages/check-website/src/helpers/sax-parser.js
Normal file
@@ -0,0 +1,14 @@
|
||||
import saxWasm from 'sax-wasm';
|
||||
import { createRequire } from 'module';
|
||||
import { readFile } from 'fs/promises';
|
||||
|
||||
export const { SaxEventType, SAXParser } = saxWasm;
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
export const streamOptions = { highWaterMark: 256 * 1024 };
|
||||
const saxPath = require.resolve('sax-wasm/lib/sax-wasm.wasm');
|
||||
const saxWasmBuffer = await readFile(saxPath);
|
||||
export const parser = new SAXParser(SaxEventType.CloseTag, streamOptions);
|
||||
|
||||
await parser.prepareWasm(saxWasmBuffer);
|
||||
14
packages/check-website/src/index.js
Normal file
14
packages/check-website/src/index.js
Normal file
@@ -0,0 +1,14 @@
|
||||
export { normalizeToLocalUrl, normalizeUrl } from './helpers/normalizeUrl.js';
|
||||
|
||||
export { CheckWebsiteCli } from './CheckWebsiteCli.js';
|
||||
|
||||
export { LocalReferencesPlugin } from './plugins/LocalReferencesPlugin.js';
|
||||
export { ExternalReferencesPlugin } from './plugins/ExternalReferencesPlugin.js';
|
||||
export { HasCanonicalPlugin } from './plugins/HasCanonicalPlugin.js';
|
||||
|
||||
export { Asset, ASSET_STATUS } from './assets/Asset.js';
|
||||
export { HtmlPage } from './assets/HtmlPage.js';
|
||||
export { AssetManager } from './assets/AssetManager.js';
|
||||
|
||||
|
||||
/** @typedef {import('../types/main.js').CheckWebsiteCliOptions} CheckWebsiteCliOptions */
|
||||
33
packages/check-website/src/issues/Issue.js
Normal file
33
packages/check-website/src/issues/Issue.js
Normal file
@@ -0,0 +1,33 @@
|
||||
export class Issue {
|
||||
options = {
|
||||
sortOrder: 0,
|
||||
duplicate: false,
|
||||
filePath: '',
|
||||
title: 'Issue',
|
||||
message: '',
|
||||
icon: '❌',
|
||||
logger: console.log,
|
||||
page: null,
|
||||
skip: false,
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {Partial<{}>} options
|
||||
*/
|
||||
constructor(options = {}) {
|
||||
this.options = { ...this.options, ...options };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {(msg: string) => void} logger
|
||||
* @returns {void}
|
||||
*/
|
||||
render(logger) {
|
||||
const useLogger = logger || this.options.logger;
|
||||
if (this.options.duplicate) {
|
||||
return;
|
||||
}
|
||||
useLogger(`${this.options.icon} ${this.options.title}: ${this.options.message}`);
|
||||
useLogger(` 🛠️ ${this.options.filePath}`);
|
||||
}
|
||||
}
|
||||
24
packages/check-website/src/issues/IssueManager.js
Normal file
24
packages/check-website/src/issues/IssueManager.js
Normal file
@@ -0,0 +1,24 @@
|
||||
export class IssueManager {
|
||||
renderErrorsOnAdd = true;
|
||||
logger = console.log;
|
||||
|
||||
/**
|
||||
* @type {(import('./Issue.js').Issue | import('./PageIssue.js').PageIssue | import('./ReferenceIssue.js').ReferenceIssue)[]}
|
||||
*/
|
||||
issues = [];
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {import('./Issue.js').Issue | import('./PageIssue.js').PageIssue | import('./ReferenceIssue.js').ReferenceIssue} issue
|
||||
*/
|
||||
add(issue) {
|
||||
this.issues.push(issue);
|
||||
if (this.renderErrorsOnAdd) {
|
||||
issue.render(this.logger);
|
||||
}
|
||||
}
|
||||
|
||||
all() {
|
||||
return this.issues;
|
||||
}
|
||||
}
|
||||
11
packages/check-website/src/issues/PageIssue.js
Normal file
11
packages/check-website/src/issues/PageIssue.js
Normal file
@@ -0,0 +1,11 @@
|
||||
import { Issue } from './Issue.js';
|
||||
|
||||
export class PageIssue extends Issue {
|
||||
constructor(options = {}) {
|
||||
super({
|
||||
message: '',
|
||||
title: 'Page Issue',
|
||||
...options,
|
||||
});
|
||||
}
|
||||
}
|
||||
11
packages/check-website/src/issues/ReferenceIssue.js
Normal file
11
packages/check-website/src/issues/ReferenceIssue.js
Normal file
@@ -0,0 +1,11 @@
|
||||
import { Issue } from './Issue.js';
|
||||
|
||||
export class ReferenceIssue extends Issue {
|
||||
constructor(options = {}) {
|
||||
super({
|
||||
title: 'Not Found',
|
||||
icon: '🔗',
|
||||
...options,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import path from 'path';
|
||||
import { cli } from '../cli/cli.js';
|
||||
import { ReferenceIssue } from '../issues/ReferenceIssue.js';
|
||||
import { Plugin } from './Plugin.js';
|
||||
|
||||
/** @typedef {import('../assets/HtmlPage.js').HtmlPage} HtmlPage */
|
||||
/** @typedef {import('../../types/main.js').Reference} Reference */
|
||||
/** @typedef {import('../../types/main.js').CheckContext} CheckContext */
|
||||
/** @typedef {import('../../types/main.js').AddToQueueHelpers} AddToQueueHelpers */
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
*/
|
||||
function getDomain(url) {
|
||||
try {
|
||||
const { hostname } = new URL(url);
|
||||
return hostname;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
export class ExternalReferencesPlugin extends Plugin {
|
||||
domainStats = new Map();
|
||||
|
||||
constructor(options = {}) {
|
||||
super({
|
||||
title: 'External',
|
||||
checkLabel: 'links',
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {CheckContext} context
|
||||
*/
|
||||
async check(context) {
|
||||
const { item, report, getAsset } = context;
|
||||
const reference = /** @type {Reference} */ (item);
|
||||
const asset = getAsset(reference.url);
|
||||
|
||||
if (!(await asset.exists())) {
|
||||
const { page } = reference;
|
||||
const relPath = path.relative(process.cwd(), page.localPath);
|
||||
const filePath = `./${relPath}:${reference.start}`;
|
||||
const message = `<${reference.tag} ${reference.attribute}="${reference.value}">`;
|
||||
report(new ReferenceIssue({ page, filePath, message }));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {HtmlPage} page
|
||||
* @param {AddToQueueHelpers} helpers
|
||||
* @returns {Promise<Reference[]>}
|
||||
*/
|
||||
async addToQueue(page, helpers) {
|
||||
const { isLocalUrl } = helpers;
|
||||
const checkItems = [];
|
||||
for (const reference of page.references) {
|
||||
if (reference.url.startsWith('http') && !isLocalUrl(reference.url)) {
|
||||
checkItems.push(reference);
|
||||
const domain = getDomain(reference.url);
|
||||
this.domainStats.set(domain, (this.domainStats.get(domain) || 0) + 1);
|
||||
}
|
||||
}
|
||||
return checkItems;
|
||||
}
|
||||
|
||||
render() {
|
||||
const top3 = [...this.domainStats.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3);
|
||||
const top3Str = top3.map(([domain, count], i) => `${i + 1}. ${domain} (${count})`).join(', ');
|
||||
return cli`
|
||||
${super.render()}
|
||||
- Top Domains: ${top3Str}
|
||||
`;
|
||||
}
|
||||
}
|
||||
60
packages/check-website/src/plugins/HasCanonicalPlugin.js
Normal file
60
packages/check-website/src/plugins/HasCanonicalPlugin.js
Normal file
@@ -0,0 +1,60 @@
|
||||
import path from 'path';
|
||||
import { PageIssue } from '../issues/PageIssue.js';
|
||||
import { Plugin } from './Plugin.js';
|
||||
|
||||
/** @typedef {import('../assets/HtmlPage.js').HtmlPage} HtmlPage */
|
||||
/** @typedef {import('../../types/main.js').CheckContext} CheckContext */
|
||||
/** @typedef {import('../../types/main.js').Reference} Reference */
|
||||
/** @typedef {import('../../types/main.js').ParseElement} ParseElement */
|
||||
|
||||
export class HasCanonicalPlugin extends Plugin {
|
||||
/**
|
||||
* Contains the unique absolute page URL as declared by the
|
||||
* `<link rel="canonical" href="...">` element (if any) for every page.
|
||||
*
|
||||
* @type {Map<HtmlPage, URL>}
|
||||
*/
|
||||
canonicalUrls = new Map();
|
||||
|
||||
/**
|
||||
* @param {ParseElement} element
|
||||
* @param {HtmlPage} page
|
||||
*/
|
||||
onParseElement = (element, page) => {
|
||||
if (!this.canonicalUrls.has(page) && element.tagName === 'LINK') {
|
||||
if (element.getAttribute('rel') === 'canonical') {
|
||||
const href = element.getAttribute('href');
|
||||
if (href) {
|
||||
this.canonicalUrls.set(page, new URL(href));
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
constructor(options = {}) {
|
||||
super({
|
||||
title: 'Canonical',
|
||||
checkLabel: 'pages',
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {CheckContext} context
|
||||
*/
|
||||
async check(context) {
|
||||
const page = /** @type {HtmlPage} */ (context.item);
|
||||
if (!this.canonicalUrls.has(page)) {
|
||||
context.report(
|
||||
new PageIssue({
|
||||
title: 'Missing canonical',
|
||||
message: 'The page is missing a <link rel="canonical" href="...">',
|
||||
page,
|
||||
filePath: './' + path.relative(process.cwd(), page.localPath),
|
||||
icon: '🦄',
|
||||
}),
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
68
packages/check-website/src/plugins/LocalReferencesPlugin.js
Normal file
68
packages/check-website/src/plugins/LocalReferencesPlugin.js
Normal file
@@ -0,0 +1,68 @@
|
||||
import normalizeUrl from 'normalize-url';
|
||||
import path from 'path';
|
||||
import { HtmlPage } from '../assets/HtmlPage.js';
|
||||
import { ReferenceIssue } from '../issues/ReferenceIssue.js';
|
||||
import { Plugin } from './Plugin.js';
|
||||
|
||||
/** @typedef {import('../../types/main.js').Reference} Reference */
|
||||
/** @typedef {import('../../types/main.js').CheckContext} CheckContext */
|
||||
/** @typedef {import('../../types/main.js').AddToQueueHelpers} AddToQueueHelpers */
|
||||
/** @typedef {import('../../types/main.js').PluginInterface} PluginInterface */
|
||||
|
||||
/**
|
||||
* @implement {PluginInterface}
|
||||
*/
|
||||
export class LocalReferencesPlugin extends Plugin {
|
||||
constructor(options = {}) {
|
||||
super({
|
||||
title: 'Local',
|
||||
checkLabel: 'links',
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {HtmlPage} page
|
||||
* @param {AddToQueueHelpers} helpers
|
||||
* @returns {Promise<Reference[]>}
|
||||
*/
|
||||
async addToQueue(page, helpers) {
|
||||
const { isLocalUrl } = helpers;
|
||||
const checkItems = [];
|
||||
for (const reference of page.references) {
|
||||
if (isLocalUrl(reference.url)) {
|
||||
checkItems.push(reference);
|
||||
}
|
||||
}
|
||||
return checkItems;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {CheckContext} context
|
||||
*/
|
||||
async check(context) {
|
||||
const { item, report, getAsset } = context;
|
||||
const reference = /** @type {Reference} */ (item);
|
||||
const targetAsset = getAsset(reference.url);
|
||||
|
||||
const { page } = reference;
|
||||
const relPath = path.relative(process.cwd(), page.localPath);
|
||||
const filePath = `./${relPath}:${reference.start}`;
|
||||
const message = `<${reference.tag} ${reference.attribute}="${reference.value}">`;
|
||||
|
||||
if (await targetAsset.exists()) {
|
||||
const urlFocusHash = normalizeUrl(reference.url, { stripTextFragment: true }); // removes :~:text=
|
||||
const hash = urlFocusHash.includes('#') ? urlFocusHash.split('#')[1] : '';
|
||||
if (
|
||||
hash &&
|
||||
targetAsset instanceof HtmlPage &&
|
||||
(await targetAsset.parse()) &&
|
||||
!targetAsset.hasHash(hash)
|
||||
) {
|
||||
report(new ReferenceIssue({ page, filePath, message, icon: '#️⃣' }));
|
||||
}
|
||||
} else {
|
||||
report(new ReferenceIssue({ page, filePath, message }));
|
||||
}
|
||||
}
|
||||
}
|
||||
215
packages/check-website/src/plugins/Plugin.js
Normal file
215
packages/check-website/src/plugins/Plugin.js
Normal file
@@ -0,0 +1,215 @@
|
||||
import { gray, green, red } from 'colorette';
|
||||
import { EventEmitter } from 'events';
|
||||
import { ASSET_STATUS } from '../assets/Asset.js';
|
||||
import { HtmlPage } from '../assets/HtmlPage.js';
|
||||
import { renderProgressBar } from '../cli/renderProgressBar.js';
|
||||
import { Queue } from '../helpers/Queue.js';
|
||||
|
||||
/** @typedef {import('../assets/Asset.js').Asset} Asset */
|
||||
/** @typedef {import('../CheckWebsiteCli.js').CheckWebsiteCli} CheckWebsiteCli */
|
||||
|
||||
/** @typedef {import('../../types/main.js').Reference} Reference */
|
||||
/** @typedef {import('../../types/main.js').CheckContext} CheckContext */
|
||||
/** @typedef {import('../../types/main.js').AddToQueueHelpers} AddToQueueHelpers */
|
||||
/** @typedef {import('../../types/main.js').PluginInterface} PluginInterface */
|
||||
|
||||
export class Plugin {
|
||||
/** @type {import('../issues/IssueManager.js').IssueManager | undefined} */
|
||||
issueManager;
|
||||
|
||||
/** @type {import('../assets/AssetManager.js').AssetManager | undefined} */
|
||||
assetManager;
|
||||
|
||||
/** @type {CheckWebsiteCli | undefined} */
|
||||
cli;
|
||||
|
||||
_passed = 0;
|
||||
_failed = 0;
|
||||
_skipped = 0;
|
||||
|
||||
/**
|
||||
* @type {[number, number] | undefined}
|
||||
*/
|
||||
_performanceStart;
|
||||
|
||||
/**
|
||||
* @type {Map<string, unknown>}
|
||||
*/
|
||||
_checkItems = new Map();
|
||||
|
||||
_queue = new Queue();
|
||||
|
||||
_processedPages = new Set();
|
||||
|
||||
/**
|
||||
* @readonly
|
||||
*/
|
||||
events = new EventEmitter();
|
||||
|
||||
/**
|
||||
* @param {Asset} asset
|
||||
*/
|
||||
async onNewParsedAsset(asset) {
|
||||
if (asset instanceof HtmlPage) {
|
||||
asset.events.on('status-changed', async () => {
|
||||
if (asset.status >= ASSET_STATUS.parsed) {
|
||||
if (!this._processedPages.has(asset)) {
|
||||
this._processedPages.add(asset);
|
||||
/** @type {AddToQueueHelpers} */
|
||||
const helpers = {
|
||||
isLocalUrl: url => this.isLocalUrl(url),
|
||||
};
|
||||
const newQueueItems = await this.addToQueue(asset, helpers);
|
||||
newQueueItems.forEach(_item => {
|
||||
this._queue.add(async () => {
|
||||
const item = /** @type {Reference | HtmlPage} */ (_item);
|
||||
|
||||
let skip = false;
|
||||
if (item.url) {
|
||||
const url = item.url instanceof URL ? item.url.href : item.url;
|
||||
const targetAsset = this.assetManager?.getAsset(url);
|
||||
if (this.isLocalUrl(url)) {
|
||||
if (targetAsset instanceof HtmlPage) {
|
||||
targetAsset.parse(); // no await but we request the parse => e.g. we crawl
|
||||
}
|
||||
}
|
||||
if (targetAsset?.options.skip) {
|
||||
skip = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (skip === false) {
|
||||
let hadIssues = false;
|
||||
/** @type {CheckContext} */
|
||||
const context = {
|
||||
report: issue => {
|
||||
hadIssues = true;
|
||||
this.issueManager?.add(issue);
|
||||
},
|
||||
item,
|
||||
getAsset: url => {
|
||||
if (!this.assetManager) {
|
||||
throw Error('Asset manager not available');
|
||||
}
|
||||
return this.assetManager.getAsset(url);
|
||||
},
|
||||
isLocalUrl: url => this.isLocalUrl(url),
|
||||
};
|
||||
await /** @type {PluginInterface} */ (/** @type {unknown} */ (this)).check(
|
||||
context,
|
||||
);
|
||||
if (hadIssues) {
|
||||
this._failed += 1;
|
||||
} else {
|
||||
this._passed += 1;
|
||||
}
|
||||
} else {
|
||||
this._skipped += 1;
|
||||
}
|
||||
|
||||
this.events.emit('progress');
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {HtmlPage} page
|
||||
* @param {AddToQueueHelpers} helpers
|
||||
* @returns {Promise<unknown[]>}
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
async addToQueue(page, helpers) {
|
||||
return [page];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Partial<{}>} options
|
||||
*/
|
||||
constructor(options = {}) {
|
||||
this.options = {
|
||||
title: 'Plugin',
|
||||
checkLabel: 'pages',
|
||||
...options,
|
||||
};
|
||||
|
||||
if (this.options.title.length > 10) {
|
||||
throw new Error(`Plugin title should be max 10 characters. Given "${this.options.title}"`);
|
||||
}
|
||||
|
||||
this._queue.on('idle', () => {
|
||||
this.events.emit('idle');
|
||||
});
|
||||
}
|
||||
|
||||
get isIdle() {
|
||||
return this._queue.isIdle;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {CheckWebsiteCli} cli
|
||||
*/
|
||||
setup(cli) {
|
||||
this._performanceStart = process.hrtime();
|
||||
this.cli = cli;
|
||||
}
|
||||
|
||||
getTotal() {
|
||||
return this._queue.getTotal();
|
||||
}
|
||||
|
||||
getDuration() {
|
||||
return this._queue.getDuration();
|
||||
}
|
||||
|
||||
getDone() {
|
||||
return this._queue.getDone();
|
||||
}
|
||||
|
||||
getPassed() {
|
||||
return this._passed;
|
||||
}
|
||||
|
||||
getFailed() {
|
||||
return this._failed;
|
||||
}
|
||||
|
||||
getSkipped() {
|
||||
return this._skipped;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @returns {boolean}
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
isLocalUrl(url) {
|
||||
return true;
|
||||
}
|
||||
|
||||
render() {
|
||||
const checkLabel = this.options.checkLabel;
|
||||
const doneNr = this.getDone();
|
||||
const passed = this.getPassed();
|
||||
const failed = this.getFailed();
|
||||
const skipped = this.getSkipped();
|
||||
const total = this.getTotal();
|
||||
|
||||
const title = `${this.options.title}:`.padEnd(11);
|
||||
const progress = renderProgressBar(doneNr, 0, total);
|
||||
|
||||
const minNumberLength = `${total}`.length;
|
||||
const done = `${doneNr}`.padStart(minNumberLength);
|
||||
|
||||
const passedTxt = passed > 0 ? `${green(`${passed} passed`)}` : '0 passed';
|
||||
const failedTxt = failed > 0 ? `, ${red(`${failed} failed`)}` : '';
|
||||
const skippedTxt = skipped > 0 ? `, ${gray(`${skipped} skipped`)}` : '';
|
||||
const resultTxt = `${passedTxt}${failedTxt}${skippedTxt}`;
|
||||
const duration = this.getDuration();
|
||||
|
||||
return `${title} ${progress} ${done}/${total} ${checkLabel} | 🕑 ${duration}s | ${resultTxt}`;
|
||||
}
|
||||
}
|
||||
56
packages/check-website/test-node/00-Asset.test.js
Normal file
56
packages/check-website/test-node/00-Asset.test.js
Normal file
@@ -0,0 +1,56 @@
|
||||
import { expect } from 'chai';
|
||||
import { HtmlPage, AssetManager } from '../src/index.js';
|
||||
|
||||
const testOptions = {
|
||||
originUrl: 'https://example.com/',
|
||||
originPath: new URL('.', import.meta.url).pathname,
|
||||
};
|
||||
|
||||
describe('Asset', () => {
|
||||
it('01: add local file via file url exists', async () => {
|
||||
const assets = new AssetManager(testOptions);
|
||||
const asset = assets.addExistingFile(
|
||||
new URL('fixtures/01-AssetManager/file.txt', import.meta.url),
|
||||
);
|
||||
expect(await asset.exists()).to.be.true;
|
||||
});
|
||||
|
||||
it('01b: local file via http url exists if added before', async () => {
|
||||
const assets = new AssetManager(testOptions);
|
||||
assets.addExistingFile(new URL('fixtures/01-AssetManager/file.txt', import.meta.url));
|
||||
const asset = assets.addUrl(new URL('https://example.com/fixtures/01-AssetManager/file.txt'));
|
||||
expect(await asset.exists()).to.be.true;
|
||||
});
|
||||
|
||||
it('01c: local file missing', async () => {
|
||||
const assets = new AssetManager(testOptions);
|
||||
const asset = assets.addUrl(
|
||||
new URL('https://example.com/fixtures/01-AssetManager/missing.txt'),
|
||||
);
|
||||
expect(await asset.exists()).to.be.false;
|
||||
});
|
||||
|
||||
it('01d: local html page exists', async () => {
|
||||
const assets = new AssetManager(testOptions);
|
||||
const page = assets.addExistingFile(
|
||||
new URL('fixtures/01-AssetManager/page.html', import.meta.url),
|
||||
);
|
||||
expect(page).to.be.an.instanceOf(HtmlPage);
|
||||
expect(await page.exists()).to.be.true;
|
||||
});
|
||||
|
||||
it('02: external file exists', async () => {
|
||||
const assets = new AssetManager(testOptions);
|
||||
const asset = assets.addUrl(new URL('https://rocket.modern-web.dev/favicon.ico'));
|
||||
expect(await asset.exists()).to.be.true;
|
||||
});
|
||||
|
||||
it('03: adds assets while parsing local pages', async () => {
|
||||
const assets = new AssetManager(testOptions);
|
||||
const page = /** @type {HtmlPage} */ (
|
||||
assets.addExistingFile(new URL('fixtures/01-AssetManager/page.html', import.meta.url))
|
||||
);
|
||||
await page.parse();
|
||||
expect(assets.size).to.equal(2);
|
||||
});
|
||||
});
|
||||
452
packages/check-website/test-node/01-HtmlPage.test.js
Normal file
452
packages/check-website/test-node/01-HtmlPage.test.js
Normal file
@@ -0,0 +1,452 @@
|
||||
import chai from 'chai';
|
||||
import path from 'path';
|
||||
import { Readable } from 'stream';
|
||||
import { readFile } from 'fs/promises';
|
||||
|
||||
import { HtmlPage, ASSET_STATUS, AssetManager } from 'check-website';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
class FakeReadable extends Readable {
|
||||
_read() {
|
||||
// noop
|
||||
}
|
||||
}
|
||||
|
||||
class MockedFetch {
|
||||
constructor({ content = '' } = {}) {
|
||||
this.stream = new FakeReadable();
|
||||
this._content = content;
|
||||
}
|
||||
|
||||
get fetch() {
|
||||
setTimeout(async () => {
|
||||
this.stream.push(this._content);
|
||||
this.stream.emit('end');
|
||||
}, 1);
|
||||
return () => Promise.resolve({ ok: true, body: this.stream });
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {any} chunk
|
||||
*/
|
||||
push(chunk) {
|
||||
return this.stream.push(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
const { expect } = chai;
|
||||
|
||||
const currentDir = path.dirname(new URL(import.meta.url).pathname);
|
||||
|
||||
/**
|
||||
* @param {HtmlPage} page
|
||||
*/
|
||||
function cleanup(page) {
|
||||
const keep = {};
|
||||
keep.hashes = page.hashes;
|
||||
keep.redirectTargetUrl = page.redirectTargetUrl;
|
||||
keep.references = page.references.map(ref => ({
|
||||
url: ref.url,
|
||||
attribute: ref.attribute,
|
||||
tag: ref.tag,
|
||||
value: ref.value,
|
||||
}));
|
||||
const optKeep = {};
|
||||
optKeep.localPath = page.options.localPath
|
||||
? `abs::${path.relative(currentDir, page.options.localPath)}`
|
||||
: page.options.localPath;
|
||||
optKeep.localSourcePath = page.options.localSourcePath
|
||||
? `abs::${path.relative(currentDir, page.options.localSourcePath)}`
|
||||
: page.options.localSourcePath;
|
||||
keep.options = optKeep;
|
||||
keep.url = page.url;
|
||||
return keep;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import('../types/main.js').HtmlPageOptions} options
|
||||
*/
|
||||
function withTestOptions(options) {
|
||||
return {
|
||||
originUrl: 'https://example.com/',
|
||||
originPath: new URL('.', import.meta.url).pathname,
|
||||
assetManager: new AssetManager(),
|
||||
...options,
|
||||
};
|
||||
}
|
||||
|
||||
describe('HtmlPage', () => {
|
||||
it('01: hashes', async () => {
|
||||
const page = new HtmlPage(
|
||||
new URL('https://example.com/fixtures/01-HtmlPage/01-hashes.html'),
|
||||
withTestOptions({
|
||||
localPath: fileURLToPath(new URL('fixtures/01-HtmlPage/01-hashes.html', import.meta.url)),
|
||||
}),
|
||||
);
|
||||
page.status = ASSET_STATUS.existsLocal;
|
||||
await page.parse();
|
||||
|
||||
expect(cleanup(page)).to.deep.equal({
|
||||
url: new URL('https://example.com/fixtures/01-HtmlPage/01-hashes.html'),
|
||||
options: {
|
||||
localPath: 'abs::fixtures/01-HtmlPage/01-hashes.html',
|
||||
localSourcePath: '',
|
||||
},
|
||||
hashes: ['first', 'second', 'third'],
|
||||
redirectTargetUrl: undefined,
|
||||
references: [
|
||||
{
|
||||
attribute: 'href',
|
||||
tag: 'a',
|
||||
value: '#',
|
||||
url: 'https://example.com/fixtures/01-HtmlPage/01-hashes.html#',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it.skip('01a: fetch it as an external url', async () => {
|
||||
const mocked = new MockedFetch({
|
||||
// @ts-ignore
|
||||
content: await readFile(new URL('fixtures/01-HtmlPage/01-hashes.html', import.meta.url)),
|
||||
});
|
||||
const page = new HtmlPage(
|
||||
new URL('https://is.mocked.com/'),
|
||||
withTestOptions({
|
||||
// @ts-ignore
|
||||
fetch: mocked.fetch,
|
||||
}),
|
||||
);
|
||||
await page.parse();
|
||||
|
||||
expect(cleanup(page)).to.deep.equal({
|
||||
url: new URL('https://is.mocked.com/'),
|
||||
options: {
|
||||
localPath: '',
|
||||
localSourcePath: '',
|
||||
},
|
||||
hashes: ['first', 'second', 'third'],
|
||||
redirectTargetUrl: undefined,
|
||||
references: [
|
||||
{
|
||||
attribute: 'href',
|
||||
tag: 'a',
|
||||
value: '#',
|
||||
url: 'https://is.mocked.com/#',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('02: internal link', async () => {
|
||||
const page = new HtmlPage(
|
||||
new URL('https://example.com/fixtures/01-HtmlPage/02-internal-link.html'),
|
||||
withTestOptions({
|
||||
localPath: fileURLToPath(
|
||||
new URL('fixtures/01-HtmlPage/02-internal-link.html', import.meta.url),
|
||||
),
|
||||
}),
|
||||
);
|
||||
page.status = ASSET_STATUS.existsLocal;
|
||||
await page.parse();
|
||||
|
||||
expect(cleanup(page)).to.deep.equal({
|
||||
url: new URL('https://example.com/fixtures/01-HtmlPage/02-internal-link.html'),
|
||||
options: {
|
||||
localPath: 'abs::fixtures/01-HtmlPage/02-internal-link.html',
|
||||
localSourcePath: '',
|
||||
},
|
||||
hashes: [],
|
||||
redirectTargetUrl: undefined,
|
||||
references: [
|
||||
{
|
||||
attribute: 'href',
|
||||
tag: 'a',
|
||||
url: 'https://example.com/index.html',
|
||||
value: '/index.html',
|
||||
},
|
||||
{
|
||||
attribute: 'href',
|
||||
tag: 'a',
|
||||
url: 'https://example.com/fixtures/01-HtmlPage/index.html',
|
||||
value: './index.html',
|
||||
},
|
||||
{
|
||||
attribute: 'href',
|
||||
tag: 'a',
|
||||
url: 'https://example.com/fixtures/01-HtmlPage/index.html#first-headline',
|
||||
value: './index.html#first-headline',
|
||||
},
|
||||
{
|
||||
attribute: 'href',
|
||||
tag: 'a',
|
||||
url: 'https://example.com/fixtures/01-HtmlPage/index.html?data=in&query=params',
|
||||
value: './index.html?data=in&query=params',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('03: images', async () => {
|
||||
const page = new HtmlPage(
|
||||
new URL('https://example.com/fixtures/01-HtmlPage/03-images.html'),
|
||||
withTestOptions({
|
||||
localPath: fileURLToPath(new URL('fixtures/01-HtmlPage/03-images.html', import.meta.url)),
|
||||
}),
|
||||
);
|
||||
page.status = ASSET_STATUS.existsLocal;
|
||||
await page.parse();
|
||||
|
||||
expect(cleanup(page)).to.deep.equal({
|
||||
url: new URL('https://example.com/fixtures/01-HtmlPage/03-images.html'),
|
||||
options: {
|
||||
localPath: 'abs::fixtures/01-HtmlPage/03-images.html',
|
||||
localSourcePath: '',
|
||||
},
|
||||
hashes: [],
|
||||
redirectTargetUrl: undefined,
|
||||
references: [
|
||||
{
|
||||
attribute: 'src',
|
||||
tag: 'img',
|
||||
url: 'https://example.com/empty.png',
|
||||
value: '/empty.png',
|
||||
},
|
||||
{
|
||||
attribute: 'src',
|
||||
tag: 'img',
|
||||
url: 'https://example.com/fixtures/01-HtmlPage/empty.png',
|
||||
value: './empty.png',
|
||||
},
|
||||
{
|
||||
attribute: 'src',
|
||||
tag: 'img',
|
||||
url: 'https://example.com/fixtures/01-HtmlPage/empty.png?data=in&query=params',
|
||||
value: './empty.png?data=in&query=params',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('04: picture with srcset', async () => {
|
||||
const page = new HtmlPage(
|
||||
new URL('https://example.com/fixtures/01-HtmlPage/04-picture-srcset.html'),
|
||||
withTestOptions({
|
||||
localPath: fileURLToPath(
|
||||
new URL('fixtures/01-HtmlPage/04-picture-srcset.html', import.meta.url),
|
||||
),
|
||||
}),
|
||||
);
|
||||
page.status = ASSET_STATUS.existsLocal;
|
||||
await page.parse();
|
||||
|
||||
expect(cleanup(page)).to.deep.equal({
|
||||
url: new URL('https://example.com/fixtures/01-HtmlPage/04-picture-srcset.html'),
|
||||
options: {
|
||||
localPath: 'abs::fixtures/01-HtmlPage/04-picture-srcset.html',
|
||||
localSourcePath: '',
|
||||
},
|
||||
hashes: [],
|
||||
redirectTargetUrl: undefined,
|
||||
references: [
|
||||
{
|
||||
attribute: 'src',
|
||||
tag: 'img',
|
||||
url: 'https://example.com/images/empty.png',
|
||||
value: '/images/empty.png',
|
||||
},
|
||||
{
|
||||
attribute: 'srcset',
|
||||
tag: 'source',
|
||||
url: 'https://example.com/images/empty-300.png',
|
||||
value: '/images/empty-300.png 300w, /images/empty-600.png?data=in&query=params 600w',
|
||||
},
|
||||
{
|
||||
attribute: 'srcset',
|
||||
tag: 'source',
|
||||
url: 'https://example.com/images/empty-600.png?data=in&query=params',
|
||||
value: '/images/empty-300.png 300w, /images/empty-600.png?data=in&query=params 600w',
|
||||
},
|
||||
{
|
||||
attribute: 'src',
|
||||
tag: 'img',
|
||||
url: 'https://example.com/images/missing.png',
|
||||
value: '/images/missing.png',
|
||||
},
|
||||
{
|
||||
attribute: 'srcset',
|
||||
tag: 'source',
|
||||
url: 'https://example.com/images/missing-300.png',
|
||||
value: '/images/missing-300.png 300w, /images/missing-600.png 600w',
|
||||
},
|
||||
{
|
||||
attribute: 'srcset',
|
||||
tag: 'source',
|
||||
url: 'https://example.com/images/missing-600.png',
|
||||
value: '/images/missing-300.png 300w, /images/missing-600.png 600w',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('05: mailto', async () => {
|
||||
const page = new HtmlPage(
|
||||
new URL('https://example.com/fixtures/01-HtmlPage/05-mailto.html'),
|
||||
withTestOptions({
|
||||
localPath: fileURLToPath(new URL('fixtures/01-HtmlPage/05-mailto.html', import.meta.url)),
|
||||
}),
|
||||
);
|
||||
page.status = ASSET_STATUS.existsLocal;
|
||||
await page.parse();
|
||||
|
||||
expect(cleanup(page)).to.deep.equal({
|
||||
url: new URL('https://example.com/fixtures/01-HtmlPage/05-mailto.html'),
|
||||
options: {
|
||||
localPath: 'abs::fixtures/01-HtmlPage/05-mailto.html',
|
||||
localSourcePath: '',
|
||||
},
|
||||
hashes: [],
|
||||
redirectTargetUrl: undefined,
|
||||
references: [
|
||||
{
|
||||
attribute: 'href',
|
||||
tag: 'a',
|
||||
url: 'mailto:foo@bar.com',
|
||||
value: 'mailto:foo@bar.com',
|
||||
},
|
||||
{
|
||||
attribute: 'href',
|
||||
tag: 'a',
|
||||
url: 'mailto:address@example.com',
|
||||
value:
|
||||
'mailto:address@example.com',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('06: not http schema', async () => {
|
||||
const page = new HtmlPage(
|
||||
new URL('https://example.com/fixtures/01-HtmlPage/06-not-http-schema.html'),
|
||||
withTestOptions({
|
||||
localPath: fileURLToPath(
|
||||
new URL('fixtures/01-HtmlPage/06-not-http-schema.html', import.meta.url),
|
||||
),
|
||||
}),
|
||||
);
|
||||
page.status = ASSET_STATUS.existsLocal;
|
||||
await page.parse();
|
||||
|
||||
expect(cleanup(page)).to.deep.equal({
|
||||
url: new URL('https://example.com/fixtures/01-HtmlPage/06-not-http-schema.html'),
|
||||
options: {
|
||||
localPath: 'abs::fixtures/01-HtmlPage/06-not-http-schema.html',
|
||||
localSourcePath: '',
|
||||
},
|
||||
hashes: [],
|
||||
redirectTargetUrl: undefined,
|
||||
references: [
|
||||
{
|
||||
attribute: 'href',
|
||||
tag: 'a',
|
||||
url: 'sketch://add-library?url=https%3A%2F%2Fmyexample.com%2Fdesign%2Fui-kit.xml',
|
||||
value: 'sketch://add-library?url=https%3A%2F%2Fmyexample.com%2Fdesign%2Fui-kit.xml',
|
||||
},
|
||||
{
|
||||
attribute: 'href',
|
||||
tag: 'a',
|
||||
url: 'vscode://file/c:/myProject/package.json:5:10',
|
||||
value: 'vscode://file/c:/myProject/package.json:5:10',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('07: tel', async () => {
|
||||
const page = new HtmlPage(
|
||||
new URL('https://example.com/fixtures/01-HtmlPage/07-tel.html'),
|
||||
withTestOptions({
|
||||
localPath: fileURLToPath(new URL('fixtures/01-HtmlPage/07-tel.html', import.meta.url)),
|
||||
}),
|
||||
);
|
||||
page.status = ASSET_STATUS.existsLocal;
|
||||
await page.parse();
|
||||
|
||||
expect(cleanup(page)).to.deep.equal({
|
||||
url: new URL('https://example.com/fixtures/01-HtmlPage/07-tel.html'),
|
||||
options: {
|
||||
localPath: 'abs::fixtures/01-HtmlPage/07-tel.html',
|
||||
localSourcePath: '',
|
||||
},
|
||||
hashes: [],
|
||||
redirectTargetUrl: undefined,
|
||||
references: [
|
||||
{
|
||||
attribute: 'href',
|
||||
tag: 'a',
|
||||
url: 'tel:99999',
|
||||
value: 'tel:99999',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('08: ignore about schema', async () => {
|
||||
const page = new HtmlPage(
|
||||
new URL('https://example.com/fixtures/01-HtmlPage/08-ignore-about-schema.html'),
|
||||
withTestOptions({
|
||||
localPath: fileURLToPath(
|
||||
new URL('fixtures/01-HtmlPage/08-ignore-about-schema.html', import.meta.url),
|
||||
),
|
||||
}),
|
||||
);
|
||||
page.status = ASSET_STATUS.existsLocal;
|
||||
await page.parse();
|
||||
|
||||
expect(cleanup(page)).to.deep.equal({
|
||||
url: new URL('https://example.com/fixtures/01-HtmlPage/08-ignore-about-schema.html'),
|
||||
options: {
|
||||
localPath: 'abs::fixtures/01-HtmlPage/08-ignore-about-schema.html',
|
||||
localSourcePath: '',
|
||||
},
|
||||
hashes: [],
|
||||
redirectTargetUrl: undefined,
|
||||
references: [
|
||||
{
|
||||
attribute: 'href',
|
||||
tag: 'a',
|
||||
url: 'about:dino',
|
||||
value: 'about:dino',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('09: html meta refresh', async () => {
|
||||
const page = new HtmlPage(
|
||||
new URL('https://example.com/fixtures/01-HtmlPage/09-html-meta-refresh.html'),
|
||||
withTestOptions({
|
||||
localPath: fileURLToPath(
|
||||
new URL('fixtures/01-HtmlPage/09-html-meta-refresh.html', import.meta.url),
|
||||
),
|
||||
}),
|
||||
);
|
||||
page.status = ASSET_STATUS.existsLocal;
|
||||
await page.parse();
|
||||
|
||||
// the HtmlPage starting it is not added as we only create it for the test
|
||||
// but the redirect it found does get added to the asset manager
|
||||
expect(page.options.assetManager?.size).to.equal(1);
|
||||
|
||||
expect(cleanup(page)).to.deep.equal({
|
||||
url: new URL('https://example.com/fixtures/01-HtmlPage/09-html-meta-refresh.html'),
|
||||
options: {
|
||||
localPath: 'abs::fixtures/01-HtmlPage/09-html-meta-refresh.html',
|
||||
localSourcePath: '',
|
||||
},
|
||||
redirectTargetUrl: new URL('https://example.com/en/getting-started'),
|
||||
hashes: [],
|
||||
references: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
68
packages/check-website/test-node/02-gatherFiles.test.js
Normal file
68
packages/check-website/test-node/02-gatherFiles.test.js
Normal file
@@ -0,0 +1,68 @@
|
||||
import { expect } from 'chai';
|
||||
import path from 'path';
|
||||
import { gatherFiles } from '../src/helpers/gatherFiles.js';
|
||||
|
||||
const currentDir = path.dirname(new URL(import.meta.url).pathname);
|
||||
|
||||
/**
|
||||
* @param {string[]} files
|
||||
*/
|
||||
function cleanupFiles(files) {
|
||||
return files.map(file => (file ? `abs::${path.relative(currentDir, file)}` : file));
|
||||
}
|
||||
|
||||
describe('gatherFiles', () => {
|
||||
it('01: current dir', async () => {
|
||||
const files = await gatherFiles(
|
||||
new URL('fixtures/02-gatherFiles/01-current-dir/', import.meta.url),
|
||||
);
|
||||
|
||||
expect(cleanupFiles(files)).to.deep.equal([
|
||||
'abs::fixtures/02-gatherFiles/01-current-dir/a.html',
|
||||
'abs::fixtures/02-gatherFiles/01-current-dir/b.js',
|
||||
'abs::fixtures/02-gatherFiles/01-current-dir/c.html',
|
||||
]);
|
||||
});
|
||||
|
||||
it('02: sub dir goes files first', async () => {
|
||||
const files = await gatherFiles(
|
||||
new URL('fixtures/02-gatherFiles/02-sub-dir/', import.meta.url),
|
||||
);
|
||||
|
||||
expect(cleanupFiles(files)).to.deep.equal([
|
||||
'abs::fixtures/02-gatherFiles/02-sub-dir/a.html',
|
||||
'abs::fixtures/02-gatherFiles/02-sub-dir/z.html',
|
||||
'abs::fixtures/02-gatherFiles/02-sub-dir/sub/b.html',
|
||||
'abs::fixtures/02-gatherFiles/02-sub-dir/sub/c.html',
|
||||
'abs::fixtures/02-gatherFiles/02-sub-dir/sub/some.js',
|
||||
'abs::fixtures/02-gatherFiles/02-sub-dir/sub2/d.html',
|
||||
]);
|
||||
});
|
||||
|
||||
it('03: index.html file always goes first', async () => {
|
||||
const files = await gatherFiles(
|
||||
new URL('fixtures/02-gatherFiles/03-index-first/', import.meta.url),
|
||||
);
|
||||
|
||||
expect(cleanupFiles(files)).to.deep.equal([
|
||||
'abs::fixtures/02-gatherFiles/03-index-first/index.html',
|
||||
'abs::fixtures/02-gatherFiles/03-index-first/about.html',
|
||||
]);
|
||||
});
|
||||
|
||||
it('03b: index.html file always goes first with sub dirs', async () => {
|
||||
const files = await gatherFiles(
|
||||
new URL('fixtures/02-gatherFiles/03b-index-first-sub-dir/', import.meta.url),
|
||||
);
|
||||
|
||||
expect(cleanupFiles(files)).to.deep.equal([
|
||||
'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/index.html',
|
||||
'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/about.html',
|
||||
'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub/index.html',
|
||||
'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub/a.html',
|
||||
'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub2/index.html',
|
||||
'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub2/b.html',
|
||||
'abs::fixtures/02-gatherFiles/03b-index-first-sub-dir/sub2/z.html',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import { expect } from 'chai';
|
||||
import { red, green } from 'colorette';
|
||||
|
||||
import { LocalReferencesPlugin } from 'check-website';
|
||||
import { setupTestCli } from './test-helpers.js';
|
||||
|
||||
function getOptions() {
|
||||
return {
|
||||
originUrl: 'https://example.com',
|
||||
plugins: [new LocalReferencesPlugin()],
|
||||
};
|
||||
}
|
||||
|
||||
describe('LocalReferencesPlugin', () => {
|
||||
it('01: finds a missing page', async () => {
|
||||
const { execute, capturedLogs } = await setupTestCli(
|
||||
'fixtures/03-LocalReferencesPlugin/01-page-missing',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
|
||||
});
|
||||
|
||||
it('01b: finds a variation of missing pages', async () => {
|
||||
const { capturedLogs, execute } = await setupTestCli(
|
||||
'fixtures/03-LocalReferencesPlugin/01b-page-missing-variations',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('6 passed')}, ${red('4 failed')}`);
|
||||
});
|
||||
|
||||
it('02: finds a missing hash', async () => {
|
||||
const { capturedLogs, execute } = await setupTestCli(
|
||||
'fixtures/03-LocalReferencesPlugin/02-hash-missing',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
|
||||
});
|
||||
|
||||
it('03: can identify full urls as internal', async () => {
|
||||
const { capturedLogs, execute } = await setupTestCli(
|
||||
'fixtures/03-LocalReferencesPlugin/03-absolute-url-as-internal',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
|
||||
});
|
||||
|
||||
it('04: automatically finds internal base url via canonical url', async () => {
|
||||
const { capturedLogs, execute } = await setupTestCli(
|
||||
'fixtures/03-LocalReferencesPlugin/04-auto-finds-internal-base-url',
|
||||
{ plugins: [new LocalReferencesPlugin()] },
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('2 passed')}, ${red('1 failed')}`);
|
||||
});
|
||||
|
||||
it('05: missing asset text file', async () => {
|
||||
const { capturedLogs, execute } = await setupTestCli(
|
||||
'fixtures/03-LocalReferencesPlugin/05-asset-missing',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
|
||||
});
|
||||
|
||||
it('06: starts crawling from the root index.html and ignores unlinked pages', async () => {
|
||||
const { capturedLogs, execute } = await setupTestCli(
|
||||
'fixtures/03-LocalReferencesPlugin/06-crawls-from-root-index',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('2 failed')}`);
|
||||
});
|
||||
|
||||
it('07: allow to mix urls with www and none www prefix', async () => {
|
||||
const { capturedLogs, execute } = await setupTestCli(
|
||||
'fixtures/03-LocalReferencesPlugin/07-mix-www-and-none-www',
|
||||
{ plugins: [new LocalReferencesPlugin()] },
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('3 passed')}, ${red('1 failed')}`);
|
||||
});
|
||||
|
||||
it('08: handle url in a case insensitive way', async () => {
|
||||
const { capturedLogs, execute } = await setupTestCli(
|
||||
'fixtures/03-LocalReferencesPlugin/08-handle-urls-case-insensitive',
|
||||
{ plugins: [new LocalReferencesPlugin()] },
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
// tests today-I-learned vs today-i-learned
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('2 passed')}, ${red('1 failed')}`);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
import { expect } from 'chai';
|
||||
import { red, green } from 'colorette';
|
||||
|
||||
import { ExternalReferencesPlugin } from 'check-website';
|
||||
import { setupTestCli } from './test-helpers.js';
|
||||
|
||||
function getOptions() {
|
||||
return {
|
||||
originUrl: 'https://example.com',
|
||||
plugins: [new ExternalReferencesPlugin()],
|
||||
};
|
||||
}
|
||||
|
||||
describe('ExternalReferencePlugin', () => {
|
||||
it('01: finds a missing page', async () => {
|
||||
const { execute, capturedLogs } = await setupTestCli(
|
||||
'fixtures/04-ExternalReferencePlugin/01-page-missing',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
|
||||
});
|
||||
|
||||
// it('02: finds a missing hash', async () => {
|
||||
// const { cli } = await setupTestCli('fixtures/04-ExternalReferencePlugin/02-hash-missing');
|
||||
// await cli.start();
|
||||
// });
|
||||
|
||||
it('03: image service using a none http url', async () => {
|
||||
const { execute, capturedLogs } = await setupTestCli(
|
||||
'fixtures/04-ExternalReferencePlugin/03-image-service-none-http',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
|
||||
});
|
||||
|
||||
it('04: data urls are not interpreted as external', async () => {
|
||||
const { execute, getLastDynamicLog } = await setupTestCli(
|
||||
'fixtures/04-ExternalReferencePlugin/04-data-urls-are-not-external',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(getLastDynamicLog()).to.include(`${green('1 passed')}, ${red('1 failed')}`);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
import { expect } from 'chai';
|
||||
import { red, green } from 'colorette';
|
||||
|
||||
import { HasCanonicalPlugin } from 'check-website';
|
||||
import { setupTestCli } from './test-helpers.js';
|
||||
|
||||
function getOptions() {
|
||||
return {
|
||||
originUrl: 'https://example.com',
|
||||
plugins: [new HasCanonicalPlugin()],
|
||||
};
|
||||
}
|
||||
|
||||
describe('HasCanonicalPlugin', () => {
|
||||
it('01: with a canonical', async () => {
|
||||
const { execute, capturedLogs } = await setupTestCli(
|
||||
'fixtures/05-HasCanonicalPlugin/01-with-canonical',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}`);
|
||||
});
|
||||
|
||||
it('02: without a canonical', async () => {
|
||||
const { execute, capturedLogs } = await setupTestCli(
|
||||
'fixtures/05-HasCanonicalPlugin/02-without-canonical',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${red('1 failed')}`);
|
||||
});
|
||||
});
|
||||
45
packages/check-website/test-node/06-config.test.js
Normal file
45
packages/check-website/test-node/06-config.test.js
Normal file
@@ -0,0 +1,45 @@
|
||||
import { expect } from 'chai';
|
||||
import { red, green, gray } from 'colorette';
|
||||
|
||||
import { LocalReferencesPlugin, ExternalReferencesPlugin } from 'check-website';
|
||||
import { setupTestCli } from './test-helpers.js';
|
||||
|
||||
function getOptions() {
|
||||
return {
|
||||
originUrl: 'https://example.com',
|
||||
plugins: [new LocalReferencesPlugin()],
|
||||
};
|
||||
}
|
||||
|
||||
describe('Config', () => {
|
||||
it('01: reads config file', async () => {
|
||||
const { execute, capturedLogs, cli } = await setupTestCli(
|
||||
'fixtures/06-config/01-change-origin-url/site',
|
||||
{ plugins: [new LocalReferencesPlugin()] },
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(cli.options.originUrl).to.equal('https://some-domain.com');
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${red('1 failed')}`);
|
||||
});
|
||||
|
||||
it('02: supports skips', async () => {
|
||||
const { execute, capturedLogs } = await setupTestCli(
|
||||
'fixtures/06-config/02-skips/site',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${green('1 passed')}, ${gray('1 skipped')}`);
|
||||
});
|
||||
|
||||
it('03: supports skips for external', async () => {
|
||||
const { execute, capturedLogs } = await setupTestCli(
|
||||
'fixtures/06-config/03-skips-external/site',
|
||||
{ originUrl: 'https://example.com', plugins: [new ExternalReferencesPlugin()] },
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
expect(capturedLogs.join('\n')).to.include(`${gray('2 skipped')}`);
|
||||
});
|
||||
});
|
||||
32
packages/check-website/test-node/07-display.test.js
Normal file
32
packages/check-website/test-node/07-display.test.js
Normal file
@@ -0,0 +1,32 @@
|
||||
import { expect } from 'chai';
|
||||
|
||||
import { LocalReferencesPlugin, ExternalReferencesPlugin } from 'check-website';
|
||||
import { setupTestCli } from './test-helpers.js';
|
||||
|
||||
function getOptions() {
|
||||
return {
|
||||
originUrl: 'https://example.com',
|
||||
plugins: [new LocalReferencesPlugin(), new ExternalReferencesPlugin()],
|
||||
};
|
||||
}
|
||||
|
||||
describe('Display', () => {
|
||||
it('01: duration is correct for multiple plugins', async () => {
|
||||
const { execute, capturedLogs } = await setupTestCli(
|
||||
'fixtures/07-display/01-duration',
|
||||
getOptions(),
|
||||
{ captureLogs: true },
|
||||
);
|
||||
await execute();
|
||||
|
||||
const lastDynamic = capturedLogs
|
||||
.join('\n')
|
||||
.split('────────────────────────────────────────────────────────────────────────────────')
|
||||
// @ts-ignore
|
||||
.at(-1);
|
||||
const matches = lastDynamic.match(/\| 1\/1 links \| 🕑 (.*)s \|/g);
|
||||
|
||||
expect(matches).to.have.lengthOf(2);
|
||||
expect(matches[0]).to.not.equal(matches[1]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1 @@
|
||||
<a href="./file.txt">File</a>
|
||||
@@ -0,0 +1,5 @@
|
||||
<h1 id="first">First</h1>
|
||||
<p>Lorem ipsum</p>
|
||||
<h2 id="second">Second</h2>
|
||||
<input name="first-name" />
|
||||
<a href="#" name="third">Third</a>
|
||||
@@ -0,0 +1,4 @@
|
||||
<a href="/index.html">index</a>
|
||||
<a href="./index.html">index</a>
|
||||
<a href="./index.html#first-headline">index</a>
|
||||
<a href="./index.html?data=in&query=params">index</a>
|
||||
@@ -0,0 +1,3 @@
|
||||
<img src="/empty.png" alt="" />
|
||||
<img src="./empty.png" alt="" />
|
||||
<img src="./empty.png?data=in&query=params" alt="" />
|
||||
@@ -0,0 +1,9 @@
|
||||
<picture>
|
||||
<source srcset="/images/empty-300.png 300w, /images/empty-600.png?data=in&query=params 600w" type="image/jpeg" sizes="(min-width: 62.5em) 25vw, (min-width: 30.625em) 50vw, 100vw">
|
||||
<img src="/images/empty.png" alt="Empty" width="300" height="225">
|
||||
</picture>
|
||||
|
||||
<picture>
|
||||
<source srcset="/images/missing-300.png 300w, /images/missing-600.png 600w" type="image/jpeg" sizes="(min-width: 62.5em) 25vw, (min-width: 30.625em) 50vw, 100vw">
|
||||
<img src="/images/missing.png" alt="Empty" width="300" height="225">
|
||||
</picture>
|
||||
@@ -0,0 +1,3 @@
|
||||
<a href="mailto:foo@bar.com"></a>
|
||||
<!-- encoded mailto links -->
|
||||
<a href="mailto:address@example.com"></a>
|
||||
@@ -0,0 +1,2 @@
|
||||
<a href="sketch://add-library?url=https%3A%2F%2Fmyexample.com%2Fdesign%2Fui-kit.xml"></a>
|
||||
<a href="vscode://file/c:/myProject/package.json:5:10"></a>
|
||||
@@ -0,0 +1 @@
|
||||
<a href="tel:99999"></a>
|
||||
@@ -0,0 +1 @@
|
||||
<a href="about:dino"></a>
|
||||
@@ -0,0 +1,2 @@
|
||||
<!DOCTYPE html>
|
||||
<meta http-equiv="refresh" content="0;url=/en/getting-started" />
|
||||
@@ -0,0 +1,2 @@
|
||||
<a href="./page.html"></a>
|
||||
<a href="./missing-page.html"></a>
|
||||
@@ -0,0 +1 @@
|
||||
<h1>Page</h1>
|
||||
@@ -0,0 +1,10 @@
|
||||
<a href="/absolute/index.html"></a>
|
||||
<a href="./relative/index.html"></a>
|
||||
|
||||
<!-- valid -->
|
||||
<a href="./page.html"></a>
|
||||
<a href=" ./page.html "></a>
|
||||
<a href=" /page.html "></a>
|
||||
<a href=""></a>
|
||||
<a href="?foo=bar&baz=now"></a>
|
||||
<a href="#:~:text=put%20your%20labels%20above%20your%20inputs">Sign-in form best practices</a>
|
||||
@@ -0,0 +1,2 @@
|
||||
<a href="/absolute-page/index.html">absolute page</a>
|
||||
<a href="./relative-page/index.html">relative page</a>
|
||||
@@ -0,0 +1,2 @@
|
||||
<a href="./page.html#first-headline"></a>
|
||||
<a href="./page.html#missing-headline"></a>
|
||||
@@ -0,0 +1 @@
|
||||
<h1 id="first-headline">First Headline</h1>
|
||||
@@ -0,0 +1,16 @@
|
||||
<a href="./page.html"></a>
|
||||
<a href="./page.html#first-headline"></a>
|
||||
<a href="./page.html#missing-headline"></a>
|
||||
<a href="./missing-page.html#missing-headline"></a>
|
||||
<a href="#local"></a>
|
||||
<a href="#local-missing"></a>
|
||||
|
||||
<p id="local"></p>
|
||||
|
||||
<a href="#audit-your-angular-app's-accessibility-with-codelyzer">
|
||||
Audit your Angular app's accessibility with codelyzer
|
||||
</a>
|
||||
|
||||
<h1 id="audit-your-angular-app's-accessibility-with-codelyzer">Audit your Angular app's accessibility with codelyzer</h1>
|
||||
|
||||
<a href="#local:~:text=put%20your%20labels%20above%20your%20inputs">Sign-in form best practices</a>
|
||||
@@ -0,0 +1 @@
|
||||
<h1 id="first-headline">First Headline</h1>
|
||||
@@ -0,0 +1,7 @@
|
||||
<!-- internal passed -->
|
||||
<a href="https://example.com/page.html"></a>
|
||||
<!-- internal failed -->
|
||||
<a href="https://example.com/missing.html"></a>
|
||||
|
||||
<!-- not internal -->
|
||||
<a href="https://modern-web.dev"></a>
|
||||
@@ -0,0 +1 @@
|
||||
<h1>Page</h1>
|
||||
@@ -0,0 +1 @@
|
||||
<h1>Page</h1>
|
||||
@@ -0,0 +1,9 @@
|
||||
<link rel="canonical" href="https://modern-web.dev" />
|
||||
|
||||
<!-- internal passed -->
|
||||
<a href="https://modern-web.dev/about.html"></a>
|
||||
<!-- internal failed -->
|
||||
<a href="https://modern-web.dev/missing.html"></a>
|
||||
|
||||
<!-- not internal -->
|
||||
<a href="https://open-wc.org"></a>
|
||||
@@ -0,0 +1,2 @@
|
||||
<a href="./info.txt"></a>
|
||||
<a href="./missing.txt"></a>
|
||||
@@ -0,0 +1 @@
|
||||
<a href="./broken-link.html">will not be found</a>
|
||||
@@ -0,0 +1,2 @@
|
||||
<a href="./page.html"></a>
|
||||
<a href="./missing-page.html"></a>
|
||||
@@ -0,0 +1,2 @@
|
||||
<h1>Page</h1>
|
||||
<a href="./missing-page.html"></a>
|
||||
@@ -0,0 +1 @@
|
||||
<h1>Page</h1>
|
||||
@@ -0,0 +1,7 @@
|
||||
<link rel="canonical" href="https://www.modern-web.dev" />
|
||||
|
||||
<!-- internal passed -->
|
||||
<a href="https://www.modern-web.dev/about.html"></a>
|
||||
<a href="https://modern-web.dev/blog/"></a>
|
||||
<!-- internal failed -->
|
||||
<a href="https://modern-web.dev/missing.html"></a>
|
||||
@@ -0,0 +1,3 @@
|
||||
<link rel="canonical" href="https://www.stefanjudis.com/" />
|
||||
<a href="/today-i-learned/top-level-await-is-available-in-node-js-modules/"></a>
|
||||
<a href="/today-i-learned/missing/"></a>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user