From af794e5fc6870cdbe59095817eacb03056910c39 Mon Sep 17 00:00:00 2001 From: Pieter Dirk Soels Date: Mon, 10 Oct 2022 14:39:31 +0200 Subject: [PATCH] Generate Markdown files from existing content for the website --- .github/workflows/deploy-website.yaml | 5 +- .../plugin/BugPatternExtractor.java | 7 +- .../plugin/BugPatternTestsExtractor.java | 3 +- .../errorprone/plugin/DocgenTaskListener.java | 8 +- .../plugin/models/BugPatternData.java | 25 ++- generate-docs.sh | 27 ---- .../refaster/test/RefasterRuleCollection.java | 6 +- website/.gitignore | 5 +- website/Gemfile | 3 + website/README.md | 13 +- website/_config.yml | 4 + website/_includes/nav_footer_custom.html | 1 + website/bug-pattern.mustache | 52 ++++++ website/generate-docs.rb | 148 ++++++++++++++++++ website/index.mustache | 7 + website/refaster-rule-collection.mustache | 56 +++++++ 16 files changed, 321 insertions(+), 49 deletions(-) delete mode 100755 generate-docs.sh create mode 100644 website/_includes/nav_footer_custom.html create mode 100644 website/bug-pattern.mustache create mode 100644 website/generate-docs.rb create mode 100644 website/index.mustache create mode 100644 website/refaster-rule-collection.mustache diff --git a/.github/workflows/deploy-website.yaml b/.github/workflows/deploy-website.yaml index 52060459..e36af558 100644 --- a/.github/workflows/deploy-website.yaml +++ b/.github/workflows/deploy-website.yaml @@ -22,11 +22,10 @@ jobs: bundler-cache: true - name: Configure Github Pages uses: actions/configure-pages@v2.1.1 + # XXX: Run website/generate-docs.rb instead. - name: Generate documentation - run: ./generate-docs.sh - - name: Build website with Jekyll working-directory: ./website - run: bundle exec jekyll build + run: bundle exec ruby generate-docs.rb - name: Validate HTML output working-directory: ./website # XXX: Drop `--disable_external true` once we fully adopted the diff --git a/docgen/src/main/java/tech/picnic/errorprone/plugin/BugPatternExtractor.java b/docgen/src/main/java/tech/picnic/errorprone/plugin/BugPatternExtractor.java index d80c4581..b0f867c2 100644 --- a/docgen/src/main/java/tech/picnic/errorprone/plugin/BugPatternExtractor.java +++ b/docgen/src/main/java/tech/picnic/errorprone/plugin/BugPatternExtractor.java @@ -1,10 +1,10 @@ package tech.picnic.errorprone.plugin; +import com.google.common.collect.ImmutableList; import com.google.errorprone.BugPattern; import com.google.errorprone.VisitorState; import com.sun.source.tree.ClassTree; import com.sun.source.util.TaskEvent; -import java.util.Arrays; import tech.picnic.errorprone.plugin.models.BugPatternData; public final class BugPatternExtractor implements DocExtractor { @@ -12,11 +12,12 @@ public final class BugPatternExtractor implements DocExtractor { public BugPatternData extractData(ClassTree tree, TaskEvent taskEvent, VisitorState state) { BugPattern annotation = taskEvent.getTypeElement().getAnnotation(BugPattern.class); return BugPatternData.create( + taskEvent.getTypeElement().getQualifiedName().toString(), taskEvent.getTypeElement().getSimpleName().toString(), - Arrays.toString(annotation.altNames()), + ImmutableList.copyOf(annotation.altNames()), annotation.linkType(), annotation.link(), - Arrays.toString(annotation.tags()), + ImmutableList.copyOf(annotation.tags()), annotation.summary(), annotation.explanation(), annotation.severity(), diff --git a/docgen/src/main/java/tech/picnic/errorprone/plugin/BugPatternTestsExtractor.java b/docgen/src/main/java/tech/picnic/errorprone/plugin/BugPatternTestsExtractor.java index 12bbe1cf..93278529 100644 --- a/docgen/src/main/java/tech/picnic/errorprone/plugin/BugPatternTestsExtractor.java +++ b/docgen/src/main/java/tech/picnic/errorprone/plugin/BugPatternTestsExtractor.java @@ -24,6 +24,7 @@ import tech.picnic.errorprone.plugin.models.BugPatternTestData; public final class BugPatternTestsExtractor implements DocExtractor { private static final Matcher JUNIT_TEST_METHOD = allOf(hasAnnotation("org.junit.jupiter.api.Test")); + private static final Matcher IDENTIFICATION_SOURCE_LINES = instanceMethod() .onDescendantOf("com.google.errorprone.CompilationTestHelper") @@ -84,7 +85,7 @@ public final class BugPatternTestsExtractor implements DocExtractor sourceLines = tree.getArguments().subList(1, tree.getArguments().size()); StringBuilder source = new StringBuilder(); diff --git a/docgen/src/main/java/tech/picnic/errorprone/plugin/DocgenTaskListener.java b/docgen/src/main/java/tech/picnic/errorprone/plugin/DocgenTaskListener.java index 5182c240..6d4afe0e 100644 --- a/docgen/src/main/java/tech/picnic/errorprone/plugin/DocgenTaskListener.java +++ b/docgen/src/main/java/tech/picnic/errorprone/plugin/DocgenTaskListener.java @@ -103,10 +103,16 @@ final class DocgenTaskListener implements TaskListener { private static boolean isBugPatternTest(ClassTree tree) { return tree.getSimpleName().toString().endsWith("Test") + // XXX: Instead, omit files from the util directory + && !tree.getSimpleName().toString().equals("MethodMatcherFactoryTest") && tree.getMembers().stream() .filter(VariableTree.class::isInstance) .map(VariableTree.class::cast) - .anyMatch(vt -> vt.getType().toString().equals("BugCheckerRefactoringTestHelper")); + .map(vt -> vt.getType().toString()) + .anyMatch( + vt -> + vt.equals("BugCheckerRefactoringTestHelper") + || vt.equals("CompilationTestHelper")); } private static String getSimpleClassName(String path) { diff --git a/docgen/src/main/java/tech/picnic/errorprone/plugin/models/BugPatternData.java b/docgen/src/main/java/tech/picnic/errorprone/plugin/models/BugPatternData.java index b943e151..6b5447f6 100644 --- a/docgen/src/main/java/tech/picnic/errorprone/plugin/models/BugPatternData.java +++ b/docgen/src/main/java/tech/picnic/errorprone/plugin/models/BugPatternData.java @@ -1,6 +1,7 @@ package tech.picnic.errorprone.plugin.models; import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; import com.google.errorprone.BugPattern.LinkType; import com.google.errorprone.BugPattern.SeverityLevel; @@ -9,30 +10,40 @@ import com.google.errorprone.BugPattern.SeverityLevel; @AutoValue public abstract class BugPatternData { public static BugPatternData create( + String fullyQualifiedName, String name, - String altNames, + ImmutableList altNames, LinkType linkType, String link, - String tags, + ImmutableList tags, String summary, String explanation, SeverityLevel severityLevel, boolean disableable) { return new AutoValue_BugPatternData( - name, altNames, linkType, link, tags, summary, explanation, severityLevel, disableable); + fullyQualifiedName, + name, + altNames, + linkType, + link, + tags, + summary, + explanation, + severityLevel, + disableable); } + abstract String fullyQualifiedName(); + abstract String name(); - // XXX: Should be `String[]`. - abstract String altNames(); + abstract ImmutableList altNames(); abstract LinkType linkType(); abstract String link(); - // XXX: Should be `String[]`. - abstract String tags(); + abstract ImmutableList tags(); abstract String summary(); diff --git a/generate-docs.sh b/generate-docs.sh deleted file mode 100755 index 1caf0f3d..00000000 --- a/generate-docs.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -set -e -u -o pipefail - -REPOSITORY_ROOT="$(git rev-parse --show-toplevel)" -WEBSITE_ROOT="${REPOSITORY_ROOT}/website" - -generate_homepage() { - local homepage="${WEBSITE_ROOT}/index.md" - - echo "Generating ${homepage}..." - cat - "${REPOSITORY_ROOT}/README.md" > "${homepage}" << EOF ---- -layout: default -title: Home -nav_order: 1 ---- -EOF - - local macos_compat="" - [[ "${OSTYPE}" == "darwin"* ]] && macos_compat="yes" - sed -i ${macos_compat:+".bak"} 's/src="website\//src="/g' "${homepage}" - sed -i ${macos_compat:+".bak"} 's/srcset="website\//srcset="/g' "${homepage}" -} - -# Generate the website. -generate_homepage diff --git a/refaster-test-support/src/main/java/tech/picnic/errorprone/refaster/test/RefasterRuleCollection.java b/refaster-test-support/src/main/java/tech/picnic/errorprone/refaster/test/RefasterRuleCollection.java index b3672045..9f0ef688 100644 --- a/refaster-test-support/src/main/java/tech/picnic/errorprone/refaster/test/RefasterRuleCollection.java +++ b/refaster-test-support/src/main/java/tech/picnic/errorprone/refaster/test/RefasterRuleCollection.java @@ -9,6 +9,7 @@ import static com.google.errorprone.BugPattern.SeverityLevel.ERROR; import static java.util.Comparator.naturalOrder; import static tech.picnic.errorprone.refaster.runner.Refaster.INCLUDED_RULES_PATTERN_FLAG; +import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; @@ -128,12 +129,13 @@ public final class RefasterRuleCollection extends BugChecker implements Compilat JavaFileObject outputFile = FileObjects.forResource(clazz, "output/" + className + "TestOutput.java"); - String inputContent, outputContent; + String inputContent; + String outputContent; try { inputContent = inputFile.getCharContent(true).toString(); outputContent = outputFile.getCharContent(true).toString(); } catch (IOException e) { - throw new RuntimeException(e); + throw new VerifyException(e); } BugCheckerRefactoringTestHelper.newInstance(RefasterRuleCollection.class, clazz) diff --git a/website/.gitignore b/website/.gitignore index 7caf6d3d..27390329 100644 --- a/website/.gitignore +++ b/website/.gitignore @@ -7,6 +7,7 @@ Gemfile.lock _site/ vendor/ -# Generated by `../generate-docs.sh`. -*.bak +# Generated by `generate-docs.rb`. index.md +bugpatterns/ +refasterrules/ diff --git a/website/Gemfile b/website/Gemfile index 75e35e6c..e99bc47b 100644 --- a/website/Gemfile +++ b/website/Gemfile @@ -1,6 +1,9 @@ ruby File.read(".ruby-version").strip source "https://rubygems.org" +gem "diffy", "~> 3.4.0" gem "github-pages", "~> 227" +gem "json", "~> 2.6" +gem "mustache", "~> 1.0" gem "rake", "~> 13.0" # Required for "just-the-docs" theme gem "webrick", "~> 1.7" diff --git a/website/README.md b/website/README.md index b9d0f983..02b0e5d0 100644 --- a/website/README.md +++ b/website/README.md @@ -2,7 +2,8 @@ This directory contains the majority of the source code that powers [error-prone.picnic.tech][error-prone-support-website]. The website is -statically generated using [Jekyll][jekyll]. +statically generated using data extracted from source code passed to +[Jekyll][jekyll] through [Mustache][mustache]. # Local development @@ -11,11 +12,12 @@ instructions][jekyll-docs-installation]. Once done, in this directory execute: ```sh bundle install -../generate-docs.sh && bundle exec jekyll serve --livereload +bundle exec ruby generate-docs.rb +bundle exec jekyll serve --livereload -I --skip-initial-build ``` The website will now be [available][localhost-port-4000] on port 4000. Source -code modifications (including the result of rerunning `../generate-docs.sh`) +code modifications (including the result of rerunning `generate-docs.rb`) will automatically be reflected. (An exception is `_config.yml`: changes to this file require a server restart.) Subsequent server restarts do not require running `bundle install`, unless `Gemfile` has been updated in the interim. @@ -24,6 +26,9 @@ If you are not familiar with Jekyll, be sure to check out its [documentation][jekyll-docs]. It is recommended to follow the provided step-by-step tutorial. +On top of Jekyll, we use the [just-the-docs][just-the-docs] theme, having +various configuration options itself. + ###### Switch Ruby versions The required Ruby version is set in `.ruby-version`. To switch, you can use @@ -56,5 +61,7 @@ Actions workflow any time a change is merged to `master`. [jekyll]: https://jekyllrb.com [jekyll-docs]: https://jekyllrb.com/docs [jekyll-docs-installation]: https://jekyllrb.com/docs/installation +[just-the-docs]: https://just-the-docs.github.io/just-the-docs/ [localhost-port-4000]: http://127.0.0.1:4000 +[mustache]: https://mustache.github.io/ [rvm]: https://rvm.io diff --git a/website/_config.yml b/website/_config.yml index 4302555b..e64ba3b2 100644 --- a/website/_config.yml +++ b/website/_config.yml @@ -12,9 +12,13 @@ plugins: # Files and directories not to be deployed through GitHub pages. exclude: + - bug-pattern.mustache - Gemfile - Gemfile.lock + - index.mustache + - generate-docs.rb - README.md + - refaster-rule-collection.mustache - vendor # See https://jekyllrb.com/docs/permalinks/#built-in-formats. diff --git a/website/_includes/nav_footer_custom.html b/website/_includes/nav_footer_custom.html new file mode 100644 index 00000000..c1b78638 --- /dev/null +++ b/website/_includes/nav_footer_custom.html @@ -0,0 +1 @@ + diff --git a/website/bug-pattern.mustache b/website/bug-pattern.mustache new file mode 100644 index 00000000..cce69561 --- /dev/null +++ b/website/bug-pattern.mustache @@ -0,0 +1,52 @@ +--- +layout: default +title: {{name}} +description: "{{{summary}}}" +parent: Bug Patterns +nav_order: 1 +--- + +# {{name}} + +{{#severity_level}} +{{{content}}} + {: .label .label-{{{color}}} } +{{/severity_level}} +{{#tags}} +{{.}} + {: .label } +{{/tags}} + +{: .summary } +{{{summary}}} + +{{{explanation}}} + +{{#suppression}} +## Suppression +{{{suppression}}} +{{/suppression}} + +{{#has_samples}} +## Samples + +{{#diff_samples}} +```diff +{{{.}}} +``` +{{/diff_samples}} + +{{#java_samples}} +```java +{{{.}}} +``` +{{/java_samples}} +{{/has_samples}} + + + View source code on GitHub + + + + diff --git a/website/generate-docs.rb b/website/generate-docs.rb new file mode 100644 index 00000000..a987f9f3 --- /dev/null +++ b/website/generate-docs.rb @@ -0,0 +1,148 @@ +#!/usr/bin/env ruby + +require 'diffy' +require 'fileutils' +require 'json' +require 'mustache' + +# XXX: Perhaps split this out into multiple Ruby files for readability. +# This script assumes to be ran inside `$repo_root/website`. +repo_root = ".." +generated_json_files_path = "#{repo_root}/error-prone-contrib/target/docs" +bug_pattern_path = "bugpatterns/" +refaster_rules_path = "refasterrules/" + +def retrieve_patterns(path) + Dir.glob("#{path}/*.json").inject({ "bug_patterns" => {}, "refaster_rules" => {} }) { |memo, file_name| + if (match = /.+(?bug-pattern|refaster).*-(?\w+?)(Test(Input|Output)?)?\.json/.match file_name) + type, name = match[:type] == 'bug-pattern' ? 'bug_patterns' : 'refaster_rules', match[:name] + memo[type][name] ||= [] + memo[type][name] << file_name + memo + else + p "Could not determine type or name of file #{file_name}" + exit 1 + end + } +end + +def parse_json(path) + File.open(path) do |file| + JSON.load(file) + end +end + +def get_bug_pattern_file_type(file) + file.end_with?('Test.json') ? "test_file" : "bug_pattern" +end + +def get_refaster_file_type(file) + if file.end_with?('TestInput.json') + 'test_input' + elsif file.end_with?('TestOutput.json') + 'test_output' + else + 'refaster_rule' + end +end + +def get_severity_color(severity) + case severity + when 'ERROR' + "red" + when 'WARNING' + "yellow" + when 'SUGGESTION' + "green" + else + "blue" + end +end + +puts 'Generating homepage...' +homepage = File.read("#{repo_root}/README.md").gsub("src=\"website/", "src=\"").gsub("srcset=\"website/", "srcset=\"") +File.write("index.md", Mustache.render(File.read("index.mustache"), homepage)) + +if !system('type mvn > /dev/null') + puts '[WARN] Could not find `mvn` on PATH. Skipping data extraction from source code of bug patterns and Refaster (test) classes.' +else + puts 'Generating JSON data for bug patterns and Refaster rules...' + # XXX: Remove `-Dvalidation.skip`. + system("mvn -f #{repo_root}/pom.xml -T1C clean install -DskipTests -Dverification.skip -Pdocgen") +end +# XXX: Rename variable, it is confusing. It is a collection of all file paths for Bug Patterns and Refaster Rules. +patterns = retrieve_patterns(generated_json_files_path) +FileUtils.mkdir_p(bug_pattern_path) +FileUtils.mkdir_p(refaster_rules_path) + +puts 'Generating bug patterns pages...' +patterns['bug_patterns'].values.each { |files| + data = files.map { |file| parse_json(file) }.inject(:merge) + + # XXX: Introduce a new `@DocumentationExample` annotation and derive samples from that. + # XXX: While awaiting the previous XXX, make a temporary decision on which and how many samples to show. + diff_samples = data['replacementTests']&.map { |testCase| Diffy::Diff.new(testCase['inputLines'], testCase['outputLines']).to_s.rstrip } || [] + java_samples = data['identificationTests']&.map { |testCase| testCase.rstrip } || [] + + # XXX: Add suppression data (once available). + render = Mustache.render(File.read("bug-pattern.mustache"), { + diff_samples: diff_samples, + explanation: data['explanation'], + has_samples: java_samples.length > 0 || diff_samples.length > 0, + java_samples: java_samples, + location: "https://github.com/PicnicSupermarket/error-prone-support/blob/master/error-prone-contrib/src/main/java/#{data['fullyQualifiedName'].gsub(/\./, "/")}.java", + name: data['name'], + severity_level: { + content: data['severityLevel'], + color: get_severity_color(data['severityLevel']) + }, + summary: data['summary'], + tags: data['tags'] + }) + File.write("#{bug_pattern_path}/#{data['name']}.md", render) +} + +puts 'Generating Refaster rules pages...' +patterns['refaster_rules'].values.each { |files| + collection_files = files.to_h { |file| [get_refaster_file_type(file), parse_json(file)] } + # XXX: Once we have rule non-test data, extract collection from there. + collection_name = collection_files['test_input']['templateCollection'] + + rules = [] + # XXX: Once we have rule non-test data, iterate over that instead and extract input like we do output now. + collection_files['test_input']["templateTests"].each { |rule| + rule_name = rule['templateName'] + # We need to strip newlines and add them to the end as a workaround to https://github.com/samg/diffy/issues/88. + input = "#{rule['templateTestContent'].strip}\n" + output = "#{collection_files['test_output']['templateTests'].find { |testCase| testCase['templateName'] == rule_name }['templateTestContent'].strip}\n" + rules << { + collection_name: collection_name, + # XXX: Derive from FQN passed when extracting RefasterRuleData, ideally with anchor info (source code line) to the inner class. + location: "https://github.com/PicnicSupermarket/error-prone-support/blob/master/error-prone-contrib/src/main/java/tech/picnic/errorprone/refasterrules/#{collection_name}.java", + rule_name: rule_name, + samples: [Diffy::Diff.new(input, output).to_s.rstrip], + # XXX: Extract suggestion and tags instead of hardcoding it. + severity_level: { + content: 'SUGGESTION', + color: get_severity_color('SUGGESTION') + }, + tags: ['Simplification'] + # XXX: Extract summary, suppression information. + } + } + + render = Mustache.render(File.read("refaster-rule-collection.mustache"), { + collection_name: collection_name, + # XXX: Derive from FQN passed when extracting RefasterRuleData. + location: "https://github.com/PicnicSupermarket/error-prone-support/blob/master/error-prone-contrib/src/main/java/tech/picnic/errorprone/refasterrules/#{collection_name}.java", + rules: rules, + severity_level: { + content: 'SUGGESTION', + color: get_severity_color('SUGGESTION') + }, + tags: ['Simplification'] }) + File.write("#{refaster_rules_path}/#{collection_name}.md", render) +} + +puts 'Generating website using Jekyll...' +system("bundle exec jekyll build") diff --git a/website/index.mustache b/website/index.mustache new file mode 100644 index 00000000..b9ad73fa --- /dev/null +++ b/website/index.mustache @@ -0,0 +1,7 @@ +--- +layout: default +title: Home +nav_order: 1 +--- + +{{{.}}} diff --git a/website/refaster-rule-collection.mustache b/website/refaster-rule-collection.mustache new file mode 100644 index 00000000..fe6f73b7 --- /dev/null +++ b/website/refaster-rule-collection.mustache @@ -0,0 +1,56 @@ +--- +layout: default +title: {{{collection_name}}} +parent: Refaster Rules +--- + +# {{{collection_name}}} +{: .no_toc } + +{{#severity_level}} +{{{content}}} + {: .label .label-{{{color}}} } +{{/severity_level}} +{{#tags}} +{{.}} + {: .label } +{{/tags}} + + + View source code on GitHub + + + + + +
+ + Table of contents + + {: .text-delta } + 1. TOC + {:toc} +
+ +{{#rules}} +## {{rule_name}} + +{{#severity_level}} +{{{content}}} + {: .label .label-{{{color}}} } +{{/severity_level}} +{{#tags}} +{{.}} + {: .label } +{{/tags}} + +### Samples +{: .no_toc .text-delta } + +{{#samples}} +```diff +{{{.}}} +``` +{{/samples}} +{{/rules}}