Generate Markdown files from existing content for the website

.github/workflows/deploy-website.yaml

@@ -22,11 +22,10 @@ jobs:
           bundler-cache: true
       - name: Configure Github Pages
         uses: actions/configure-pages@v2.1.1
+      # XXX: Run website/generate-docs.rb instead.
       - name: Generate documentation
-        run: ./generate-docs.sh
-      - name: Build website with Jekyll
         working-directory: ./website
-        run: bundle exec jekyll build
+        run: bundle exec ruby generate-docs.rb
       - name: Validate HTML output
         working-directory: ./website
         # XXX: Drop `--disable_external true` once we fully adopted the

docgen/src/main/java/tech/picnic/errorprone/plugin/BugPatternExtractor.java

@@ -1,10 +1,10 @@
 package tech.picnic.errorprone.plugin;
 
+import com.google.common.collect.ImmutableList;
 import com.google.errorprone.BugPattern;
 import com.google.errorprone.VisitorState;
 import com.sun.source.tree.ClassTree;
 import com.sun.source.util.TaskEvent;
-import java.util.Arrays;
 import tech.picnic.errorprone.plugin.models.BugPatternData;
 
 public final class BugPatternExtractor implements DocExtractor<BugPatternData> {
@@ -12,11 +12,12 @@ public final class BugPatternExtractor implements DocExtractor<BugPatternData> {
   public BugPatternData extractData(ClassTree tree, TaskEvent taskEvent, VisitorState state) {
     BugPattern annotation = taskEvent.getTypeElement().getAnnotation(BugPattern.class);
     return BugPatternData.create(
+        taskEvent.getTypeElement().getQualifiedName().toString(),
         taskEvent.getTypeElement().getSimpleName().toString(),
-        Arrays.toString(annotation.altNames()),
+        ImmutableList.copyOf(annotation.altNames()),
         annotation.linkType(),
         annotation.link(),
-        Arrays.toString(annotation.tags()),
+        ImmutableList.copyOf(annotation.tags()),
         annotation.summary(),
         annotation.explanation(),
         annotation.severity(),

docgen/src/main/java/tech/picnic/errorprone/plugin/BugPatternTestsExtractor.java

@@ -24,6 +24,7 @@ import tech.picnic.errorprone.plugin.models.BugPatternTestData;
 public final class BugPatternTestsExtractor implements DocExtractor<BugPatternTestData> {
   private static final Matcher<MethodTree> JUNIT_TEST_METHOD =
       allOf(hasAnnotation("org.junit.jupiter.api.Test"));
+
   private static final Matcher<ExpressionTree> IDENTIFICATION_SOURCE_LINES =
       instanceMethod()
           .onDescendantOf("com.google.errorprone.CompilationTestHelper")
@@ -84,7 +85,7 @@ public final class BugPatternTestsExtractor implements DocExtractor<BugPatternTe
       return super.visitMethodInvocation(node, unused);
     }
 
-    private String getSourceLines(MethodInvocationTree tree) {
+    private static String getSourceLines(MethodInvocationTree tree) {
       List<? extends ExpressionTree> sourceLines =
           tree.getArguments().subList(1, tree.getArguments().size());
       StringBuilder source = new StringBuilder();

docgen/src/main/java/tech/picnic/errorprone/plugin/DocgenTaskListener.java

@@ -103,10 +103,16 @@ final class DocgenTaskListener implements TaskListener {
 
   private static boolean isBugPatternTest(ClassTree tree) {
     return tree.getSimpleName().toString().endsWith("Test")
+        // XXX: Instead, omit files from the util directory
+        && !tree.getSimpleName().toString().equals("MethodMatcherFactoryTest")
         && tree.getMembers().stream()
             .filter(VariableTree.class::isInstance)
             .map(VariableTree.class::cast)
-            .anyMatch(vt -> vt.getType().toString().equals("BugCheckerRefactoringTestHelper"));
+            .map(vt -> vt.getType().toString())
+            .anyMatch(
+                vt ->
+                    vt.equals("BugCheckerRefactoringTestHelper")
+                        || vt.equals("CompilationTestHelper"));
   }
 
   private static String getSimpleClassName(String path) {

docgen/src/main/java/tech/picnic/errorprone/plugin/models/BugPatternData.java

@@ -1,6 +1,7 @@
 package tech.picnic.errorprone.plugin.models;
 
 import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
 import com.google.errorprone.BugPattern.LinkType;
 import com.google.errorprone.BugPattern.SeverityLevel;
 
@@ -9,30 +10,40 @@ import com.google.errorprone.BugPattern.SeverityLevel;
 @AutoValue
 public abstract class BugPatternData {
   public static BugPatternData create(
+      String fullyQualifiedName,
       String name,
-      String altNames,
+      ImmutableList<String> altNames,
       LinkType linkType,
       String link,
-      String tags,
+      ImmutableList<String> tags,
       String summary,
       String explanation,
       SeverityLevel severityLevel,
       boolean disableable) {
     return new AutoValue_BugPatternData(
-        name, altNames, linkType, link, tags, summary, explanation, severityLevel, disableable);
+        fullyQualifiedName,
+        name,
+        altNames,
+        linkType,
+        link,
+        tags,
+        summary,
+        explanation,
+        severityLevel,
+        disableable);
   }
 
+  abstract String fullyQualifiedName();
+
   abstract String name();
 
-  // XXX: Should be `String[]`.
-  abstract String altNames();
+  abstract ImmutableList<String> altNames();
 
   abstract LinkType linkType();
 
   abstract String link();
 
-  // XXX: Should be `String[]`.
-  abstract String tags();
+  abstract ImmutableList<String> tags();
 
   abstract String summary();
 

generate-docs.sh

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-set -e -u -o pipefail
-
-REPOSITORY_ROOT="$(git rev-parse --show-toplevel)"
-WEBSITE_ROOT="${REPOSITORY_ROOT}/website"
-
-generate_homepage() {
-    local homepage="${WEBSITE_ROOT}/index.md"
-
-    echo "Generating ${homepage}..."
-    cat - "${REPOSITORY_ROOT}/README.md" > "${homepage}" << EOF
----
-layout: default
-title: Home
-nav_order: 1
----
-EOF
-
-    local macos_compat=""
-    [[ "${OSTYPE}" == "darwin"* ]] && macos_compat="yes"
-    sed -i ${macos_compat:+".bak"} 's/src="website\//src="/g' "${homepage}"
-    sed -i ${macos_compat:+".bak"} 's/srcset="website\//srcset="/g' "${homepage}"
-}
-
-# Generate the website.
-generate_homepage

refaster-test-support/src/main/java/tech/picnic/errorprone/refaster/test/RefasterRuleCollection.java

@@ -9,6 +9,7 @@ import static com.google.errorprone.BugPattern.SeverityLevel.ERROR;
 import static java.util.Comparator.naturalOrder;
 import static tech.picnic.errorprone.refaster.runner.Refaster.INCLUDED_RULES_PATTERN_FLAG;
 
+import com.google.common.base.VerifyException;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ImmutableMap;
@@ -128,12 +129,13 @@ public final class RefasterRuleCollection extends BugChecker implements Compilat
     JavaFileObject outputFile =
         FileObjects.forResource(clazz, "output/" + className + "TestOutput.java");
 
-    String inputContent, outputContent;
+    String inputContent;
+    String outputContent;
     try {
       inputContent = inputFile.getCharContent(true).toString();
       outputContent = outputFile.getCharContent(true).toString();
     } catch (IOException e) {
-      throw new RuntimeException(e);
+      throw new VerifyException(e);
     }
 
     BugCheckerRefactoringTestHelper.newInstance(RefasterRuleCollection.class, clazz)

website/.gitignore

@@ -7,6 +7,7 @@ Gemfile.lock
 _site/
 vendor/
 
-# Generated by `../generate-docs.sh`.
-*.bak
+# Generated by `generate-docs.rb`.
 index.md
+bugpatterns/
+refasterrules/

website/Gemfile

@@ -1,6 +1,9 @@
 ruby File.read(".ruby-version").strip
 
 source "https://rubygems.org"
+gem "diffy", "~> 3.4.0"
 gem "github-pages", "~> 227"
+gem "json", "~> 2.6"
+gem "mustache", "~> 1.0"
 gem "rake", "~> 13.0" # Required for "just-the-docs" theme
 gem "webrick", "~> 1.7"

website/README.md

@@ -2,7 +2,8 @@
 
 This directory contains the majority of the source code that powers
 [error-prone.picnic.tech][error-prone-support-website]. The website is
-statically generated using [Jekyll][jekyll].
+statically generated using data extracted from source code passed to
+[Jekyll][jekyll] through [Mustache][mustache].
 
 # Local development
 
@@ -11,11 +12,12 @@ instructions][jekyll-docs-installation]. Once done, in this directory execute:
 
 ```sh
 bundle install
-../generate-docs.sh && bundle exec jekyll serve --livereload
+bundle exec ruby generate-docs.rb
+bundle exec jekyll serve --livereload -I --skip-initial-build
 ```
 
 The website will now be [available][localhost-port-4000] on port 4000. Source
-code modifications (including the result of rerunning `../generate-docs.sh`)
+code modifications (including the result of rerunning `generate-docs.rb`)
 will automatically be reflected. (An exception is `_config.yml`: changes to
 this file require a server restart.) Subsequent server restarts do not require
 running `bundle install`, unless `Gemfile` has been updated in the interim.
@@ -24,6 +26,9 @@ If you are not familiar with Jekyll, be sure to check out its
 [documentation][jekyll-docs]. It is recommended to follow the provided
 step-by-step tutorial.
 
+On top of Jekyll, we use the [just-the-docs][just-the-docs] theme, having
+various configuration options itself.
+
 ###### Switch Ruby versions
 
 The required Ruby version is set in `.ruby-version`. To switch, you can use
@@ -56,5 +61,7 @@ Actions workflow any time a change is merged to `master`.
 [jekyll]: https://jekyllrb.com
 [jekyll-docs]: https://jekyllrb.com/docs
 [jekyll-docs-installation]: https://jekyllrb.com/docs/installation
+[just-the-docs]: https://just-the-docs.github.io/just-the-docs/
 [localhost-port-4000]: http://127.0.0.1:4000
+[mustache]: https://mustache.github.io/
 [rvm]: https://rvm.io

website/_config.yml

@@ -12,9 +12,13 @@ plugins:
 
 # Files and directories not to be deployed through GitHub pages.
 exclude:
+  - bug-pattern.mustache
   - Gemfile
   - Gemfile.lock
+  - index.mustache
+  - generate-docs.rb
   - README.md
+  - refaster-rule-collection.mustache
   - vendor
 
 # See https://jekyllrb.com/docs/permalinks/#built-in-formats.

website/_includes/nav_footer_custom.html

@@ -0,0 +1 @@
+<!-- This file removes the just-the-docs reference in the nav bar's footer. -->

website/bug-pattern.mustache

@@ -0,0 +1,52 @@
+---
+layout: default
+title: {{name}}
+description: "{{{summary}}}"
+parent: Bug Patterns
+nav_order: 1
+---
+
+# {{name}}
+
+{{#severity_level}}
+{{{content}}}
+  {: .label .label-{{{color}}} }
+{{/severity_level}}
+{{#tags}}
+{{.}}
+  {: .label }
+{{/tags}}
+
+{: .summary }
+{{{summary}}}
+
+{{{explanation}}}
+
+{{#suppression}}
+## Suppression
+{{{suppression}}}
+{{/suppression}}
+
+{{#has_samples}}
+## Samples
+
+{{#diff_samples}}
+```diff
+{{{.}}}
+```
+{{/diff_samples}}
+
+{{#java_samples}}
+```java
+{{{.}}}
+```
+{{/java_samples}}
+{{/has_samples}}
+
+<a href="https://github.com/PicnicSupermarket/error-prone-support/blob/master/error-prone-contrib/src/main/java/{{bugpattern}}" class="fs-3 btn external"
+   target="_blank">
+    View source code on GitHub
+    <svg viewBox="0 0 24 24" aria-labelledby="svg-external-link-title">
+        <use xlink:href="#svg-external-link"></use>
+    </svg>
+</a>

website/generate-docs.rb

@@ -0,0 +1,148 @@
+#!/usr/bin/env ruby
+
+require 'diffy'
+require 'fileutils'
+require 'json'
+require 'mustache'
+
+# XXX: Perhaps split this out into multiple Ruby files for readability.
+# This script assumes to be ran inside `$repo_root/website`.
+repo_root = ".."
+generated_json_files_path = "#{repo_root}/error-prone-contrib/target/docs"
+bug_pattern_path = "bugpatterns/"
+refaster_rules_path = "refasterrules/"
+
+def retrieve_patterns(path)
+  Dir.glob("#{path}/*.json").inject({ "bug_patterns" => {}, "refaster_rules" => {} }) { |memo, file_name|
+    if (match = /.+(?<type>bug-pattern|refaster).*-(?<name>\w+?)(Test(Input|Output)?)?\.json/.match file_name)
+      type, name = match[:type] == 'bug-pattern' ? 'bug_patterns' : 'refaster_rules', match[:name]
+      memo[type][name] ||= []
+      memo[type][name] << file_name
+      memo
+    else
+      p "Could not determine type or name of file #{file_name}"
+      exit 1
+    end
+  }
+end
+
+def parse_json(path)
+  File.open(path) do |file|
+    JSON.load(file)
+  end
+end
+
+def get_bug_pattern_file_type(file)
+  file.end_with?('Test.json') ? "test_file" : "bug_pattern"
+end
+
+def get_refaster_file_type(file)
+  if file.end_with?('TestInput.json')
+    'test_input'
+  elsif file.end_with?('TestOutput.json')
+    'test_output'
+  else
+    'refaster_rule'
+  end
+end
+
+def get_severity_color(severity)
+  case severity
+  when 'ERROR'
+    "red"
+  when 'WARNING'
+    "yellow"
+  when 'SUGGESTION'
+    "green"
+  else
+    "blue"
+  end
+end
+
+puts 'Generating homepage...'
+homepage = File.read("#{repo_root}/README.md").gsub("src=\"website/", "src=\"").gsub("srcset=\"website/", "srcset=\"")
+File.write("index.md", Mustache.render(File.read("index.mustache"), homepage))
+
+if !system('type mvn > /dev/null')
+  puts '[WARN] Could not find `mvn` on PATH. Skipping data extraction from source code of bug patterns and Refaster (test) classes.'
+else
+  puts 'Generating JSON data for bug patterns and Refaster rules...'
+  # XXX: Remove `-Dvalidation.skip`.
+  system("mvn -f #{repo_root}/pom.xml -T1C clean install -DskipTests -Dverification.skip -Pdocgen")
+end
+# XXX: Rename variable, it is confusing. It is a collection of all file paths for Bug Patterns and Refaster Rules.
+patterns = retrieve_patterns(generated_json_files_path)
+FileUtils.mkdir_p(bug_pattern_path)
+FileUtils.mkdir_p(refaster_rules_path)
+
+puts 'Generating bug patterns pages...'
+patterns['bug_patterns'].values.each { |files|
+  data = files.map { |file| parse_json(file) }.inject(:merge)
+
+  # XXX: Introduce a new `@DocumentationExample` annotation and derive samples from that.
+  # XXX: While awaiting the previous XXX, make a temporary decision on which and how many samples to show.
+  diff_samples = data['replacementTests']&.map { |testCase| Diffy::Diff.new(testCase['inputLines'], testCase['outputLines']).to_s.rstrip } || []
+  java_samples = data['identificationTests']&.map { |testCase| testCase.rstrip } || []
+
+  # XXX: Add suppression data (once available).
+  render = Mustache.render(File.read("bug-pattern.mustache"), {
+    diff_samples: diff_samples,
+    explanation: data['explanation'],
+    has_samples: java_samples.length > 0 || diff_samples.length > 0,
+    java_samples: java_samples,
+    location: "https://github.com/PicnicSupermarket/error-prone-support/blob/master/error-prone-contrib/src/main/java/#{data['fullyQualifiedName'].gsub(/\./, "/")}.java",
+    name: data['name'],
+    severity_level: {
+      content: data['severityLevel'],
+      color: get_severity_color(data['severityLevel'])
+    },
+    summary: data['summary'],
+    tags: data['tags']
+  })
+  File.write("#{bug_pattern_path}/#{data['name']}.md", render)
+}
+
+puts 'Generating Refaster rules pages...'
+patterns['refaster_rules'].values.each { |files|
+  collection_files = files.to_h { |file| [get_refaster_file_type(file), parse_json(file)] }
+  # XXX: Once we have rule non-test data, extract collection from there.
+  collection_name = collection_files['test_input']['templateCollection']
+
+  rules = []
+  # XXX: Once we have rule non-test data, iterate over that instead and extract input like we do output now.
+  collection_files['test_input']["templateTests"].each { |rule|
+    rule_name = rule['templateName']
+    # We need to strip newlines and add them to the end as a workaround to https://github.com/samg/diffy/issues/88.
+    input = "#{rule['templateTestContent'].strip}\n"
+    output = "#{collection_files['test_output']['templateTests'].find { |testCase| testCase['templateName'] == rule_name }['templateTestContent'].strip}\n"
+    rules << {
+      collection_name: collection_name,
+      # XXX: Derive from FQN passed when extracting RefasterRuleData, ideally with anchor info (source code line) to the inner class.
+      location: "https://github.com/PicnicSupermarket/error-prone-support/blob/master/error-prone-contrib/src/main/java/tech/picnic/errorprone/refasterrules/#{collection_name}.java",
+      rule_name: rule_name,
+      samples: [Diffy::Diff.new(input, output).to_s.rstrip],
+      # XXX: Extract suggestion and tags instead of hardcoding it.
+      severity_level: {
+        content: 'SUGGESTION',
+        color: get_severity_color('SUGGESTION')
+      },
+      tags: ['Simplification']
+      # XXX: Extract summary, suppression information.
+    }
+  }
+
+  render = Mustache.render(File.read("refaster-rule-collection.mustache"), {
+    collection_name: collection_name,
+    # XXX: Derive from FQN passed when extracting RefasterRuleData.
+    location: "https://github.com/PicnicSupermarket/error-prone-support/blob/master/error-prone-contrib/src/main/java/tech/picnic/errorprone/refasterrules/#{collection_name}.java",
+    rules: rules,
+    severity_level: {
+      content: 'SUGGESTION',
+      color: get_severity_color('SUGGESTION')
+    },
+    tags: ['Simplification'] })
+  File.write("#{refaster_rules_path}/#{collection_name}.md", render)
+}
+
+puts 'Generating website using Jekyll...'
+system("bundle exec jekyll build")

website/index.mustache

@@ -0,0 +1,7 @@
+---
+layout: default
+title: Home
+nav_order: 1
+---
+
+{{{.}}}

website/refaster-rule-collection.mustache

@@ -0,0 +1,56 @@
+---
+layout: default
+title: {{{collection_name}}}
+parent: Refaster Rules
+---
+
+# {{{collection_name}}}
+{: .no_toc }
+
+{{#severity_level}}
+{{{content}}}
+  {: .label .label-{{{color}}} }
+{{/severity_level}}
+{{#tags}}
+{{.}}
+  {: .label }
+{{/tags}}
+
+<a href="{{location}}" class="fs-3 btn external"
+   target="_blank">
+    View source code on GitHub
+    <svg viewBox="0 0 24 24" aria-labelledby="svg-external-link-title">
+        <use xlink:href="#svg-external-link"></use>
+    </svg>
+</a>
+
+<details open markdown="block">
+  <summary>
+    Table of contents
+  </summary>
+  {: .text-delta }
+  1. TOC
+  {:toc}
+</details>
+
+{{#rules}}
+## {{rule_name}}
+
+{{#severity_level}}
+{{{content}}}
+  {: .label .label-{{{color}}} }
+{{/severity_level}}
+{{#tags}}
+{{.}}
+  {: .label }
+{{/tags}}
+
+### Samples
+{: .no_toc .text-delta }
+
+{{#samples}}
+```diff
+{{{.}}}
+```
+{{/samples}}
+{{/rules}}