Introduce RefasterRuleTestExtractor for documentation generation (#1317)

This new `Extractor` implementation collects Refaster example input and 
output code from rule collection tests.

This change also introduces explicit compilation steps for the test
code. As a side-effect this produces faster feedback in case of invalid
input or output code.

documentation-support/pom.xml

@@ -33,6 +33,15 @@
             <artifactId>error_prone_test_helpers</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>error-prone-utils</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>refaster-test-support</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>

documentation-support/src/main/java/tech/picnic/errorprone/documentation/BugPatternTestExtractor.java

@@ -27,7 +27,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import org.jspecify.annotations.Nullable;
-import tech.picnic.errorprone.documentation.BugPatternTestExtractor.TestCases;
+import tech.picnic.errorprone.documentation.BugPatternTestExtractor.BugPatternTestCases;
 
 /**
  * An {@link Extractor} that describes how to extract data from classes that test a {@code
@@ -40,7 +40,7 @@ import tech.picnic.errorprone.documentation.BugPatternTestExtractor.TestCases;
 @Immutable
 @AutoService(Extractor.class)
 @SuppressWarnings("rawtypes" /* See https://github.com/google/auto/issues/870. */)
-public final class BugPatternTestExtractor implements Extractor<TestCases> {
+public final class BugPatternTestExtractor implements Extractor<BugPatternTestCases> {
   /** Instantiates a new {@link BugPatternTestExtractor} instance. */
   public BugPatternTestExtractor() {}
 
@@ -50,7 +50,7 @@ public final class BugPatternTestExtractor implements Extractor<TestCases> {
   }
 
   @Override
-  public Optional<TestCases> tryExtract(ClassTree tree, VisitorState state) {
+  public Optional<BugPatternTestCases> tryExtract(ClassTree tree, VisitorState state) {
     BugPatternTestCollector collector = new BugPatternTestCollector();
 
     collector.scan(tree, state);
@@ -59,7 +59,7 @@ public final class BugPatternTestExtractor implements Extractor<TestCases> {
         .filter(not(ImmutableList::isEmpty))
         .map(
             tests ->
-                new AutoValue_BugPatternTestExtractor_TestCases(
+                new AutoValue_BugPatternTestExtractor_BugPatternTestCases(
                     state.getPath().getCompilationUnit().getSourceFile().toUri(),
                     ASTHelpers.getSymbol(tree).className(),
                     tests));
@@ -95,10 +95,10 @@ public final class BugPatternTestExtractor implements Extractor<TestCases> {
             .onDescendantOf("com.google.errorprone.BugCheckerRefactoringTestHelper.ExpectOutput")
             .namedAnyOf("addOutputLines", "expectUnchanged");
 
-    private final List<TestCase> collectedTestCases = new ArrayList<>();
+    private final List<BugPatternTestCase> collectedBugPatternTestCases = new ArrayList<>();
 
-    private ImmutableList<TestCase> getCollectedTests() {
-      return ImmutableList.copyOf(collectedTestCases);
+    private ImmutableList<BugPatternTestCase> getCollectedTests() {
+      return ImmutableList.copyOf(collectedBugPatternTestCases);
     }
 
     @Override
@@ -110,14 +110,14 @@ public final class BugPatternTestExtractor implements Extractor<TestCases> {
                 classUnderTest -> {
                   List<TestEntry> entries = new ArrayList<>();
                   if (isReplacementTest) {
-                    extractReplacementTestCases(node, entries, state);
+                    extractReplacementBugPatternTestCases(node, entries, state);
                   } else {
-                    extractIdentificationTestCases(node, entries, state);
+                    extractIdentificationBugPatternTestCases(node, entries, state);
                   }
 
                   if (!entries.isEmpty()) {
-                    collectedTestCases.add(
-                        new AutoValue_BugPatternTestExtractor_TestCase(
+                    collectedBugPatternTestCases.add(
+                        new AutoValue_BugPatternTestExtractor_BugPatternTestCase(
                             classUnderTest, ImmutableList.copyOf(entries).reverse()));
                   }
                 });
@@ -140,7 +140,7 @@ public final class BugPatternTestExtractor implements Extractor<TestCases> {
           : Optional.empty();
     }
 
-    private static void extractIdentificationTestCases(
+    private static void extractIdentificationBugPatternTestCases(
         MethodInvocationTree tree, List<TestEntry> sink, VisitorState state) {
       if (IDENTIFICATION_SOURCE_LINES.matches(tree, state)) {
         String path = ASTHelpers.constValue(tree.getArguments().get(0), String.class);
@@ -155,11 +155,11 @@ public final class BugPatternTestExtractor implements Extractor<TestCases> {
 
       ExpressionTree receiver = ASTHelpers.getReceiver(tree);
       if (receiver instanceof MethodInvocationTree methodInvocation) {
-        extractIdentificationTestCases(methodInvocation, sink, state);
+        extractIdentificationBugPatternTestCases(methodInvocation, sink, state);
       }
     }
 
-    private static void extractReplacementTestCases(
+    private static void extractReplacementBugPatternTestCases(
         MethodInvocationTree tree, List<TestEntry> sink, VisitorState state) {
       if (REPLACEMENT_OUTPUT_SOURCE_LINES.matches(tree, state)) {
         /*
@@ -185,7 +185,7 @@ public final class BugPatternTestExtractor implements Extractor<TestCases> {
 
       ExpressionTree receiver = ASTHelpers.getReceiver(tree);
       if (receiver instanceof MethodInvocationTree methodInvocation) {
-        extractReplacementTestCases(methodInvocation, sink, state);
+        extractReplacementBugPatternTestCases(methodInvocation, sink, state);
       }
     }
 
@@ -208,24 +208,26 @@ public final class BugPatternTestExtractor implements Extractor<TestCases> {
   }
 
   @AutoValue
-  @JsonDeserialize(as = AutoValue_BugPatternTestExtractor_TestCases.class)
-  abstract static class TestCases {
-    static TestCases create(URI source, String testClass, ImmutableList<TestCase> testCases) {
-      return new AutoValue_BugPatternTestExtractor_TestCases(source, testClass, testCases);
+  @JsonDeserialize(as = AutoValue_BugPatternTestExtractor_BugPatternTestCases.class)
+  abstract static class BugPatternTestCases {
+    static BugPatternTestCases create(
+        URI source, String testClass, ImmutableList<BugPatternTestCase> testCases) {
+      return new AutoValue_BugPatternTestExtractor_BugPatternTestCases(
+          source, testClass, testCases);
     }
 
     abstract URI source();
 
     abstract String testClass();
 
-    abstract ImmutableList<TestCase> testCases();
+    abstract ImmutableList<BugPatternTestCase> testCases();
   }
 
   @AutoValue
-  @JsonDeserialize(as = AutoValue_BugPatternTestExtractor_TestCase.class)
-  abstract static class TestCase {
-    static TestCase create(String classUnderTest, ImmutableList<TestEntry> entries) {
-      return new AutoValue_BugPatternTestExtractor_TestCase(classUnderTest, entries);
+  @JsonDeserialize(as = AutoValue_BugPatternTestExtractor_BugPatternTestCase.class)
+  abstract static class BugPatternTestCase {
+    static BugPatternTestCase create(String classUnderTest, ImmutableList<TestEntry> entries) {
+      return new AutoValue_BugPatternTestExtractor_BugPatternTestCase(classUnderTest, entries);
     }
 
     abstract String classUnderTest();

documentation-support/src/main/java/tech/picnic/errorprone/documentation/RefasterRuleCollectionTestExtractor.java

@@ -0,0 +1,176 @@
+package tech.picnic.errorprone.documentation;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.errorprone.matchers.Matchers.isSubtypeOf;
+import static java.util.stream.Collectors.joining;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.google.auto.service.AutoService;
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Splitter;
+import com.google.common.base.Supplier;
+import com.google.common.base.VerifyException;
+import com.google.common.collect.ImmutableList;
+import com.google.errorprone.VisitorState;
+import com.google.errorprone.annotations.FormatMethod;
+import com.google.errorprone.annotations.Immutable;
+import com.google.errorprone.matchers.Matcher;
+import com.sun.source.tree.ClassTree;
+import com.sun.source.tree.MethodTree;
+import java.net.URI;
+import java.util.Optional;
+import java.util.regex.Pattern;
+import tech.picnic.errorprone.documentation.RefasterRuleCollectionTestExtractor.RefasterTestCases;
+import tech.picnic.errorprone.utils.SourceCode;
+
+/**
+ * An {@link Extractor} that describes how to extract data from Refaster rule input and output test
+ * classes.
+ */
+// XXX: Drop this extractor if/when the Refaster test framework is reimplemented such that tests can
+// be located alongside rules, rather than in two additional resource files as currently required by
+// `RefasterRuleCollection`.
+@Immutable
+@AutoService(Extractor.class)
+@SuppressWarnings("rawtypes" /* See https://github.com/google/auto/issues/870. */)
+public final class RefasterRuleCollectionTestExtractor implements Extractor<RefasterTestCases> {
+  private static final Matcher<ClassTree> IS_REFASTER_RULE_COLLECTION_TEST_CASE =
+      isSubtypeOf("tech.picnic.errorprone.refaster.test.RefasterRuleCollectionTestCase");
+  private static final Pattern TEST_CLASS_NAME_PATTERN = Pattern.compile("(.*)Test");
+  private static final Pattern TEST_CLASS_FILE_NAME_PATTERN =
+      Pattern.compile(".*(Input|Output)\\.java");
+  private static final Pattern TEST_METHOD_NAME_PATTERN = Pattern.compile("test(.*)");
+  private static final String LINE_SEPARATOR = "\n";
+  private static final Splitter LINE_SPLITTER = Splitter.on(LINE_SEPARATOR);
+
+  /** Instantiates a new {@link RefasterRuleCollectionTestExtractor} instance. */
+  public RefasterRuleCollectionTestExtractor() {}
+
+  @Override
+  public String identifier() {
+    return "refaster-rule-collection-test";
+  }
+
+  @Override
+  public Optional<RefasterTestCases> tryExtract(ClassTree tree, VisitorState state) {
+    if (!IS_REFASTER_RULE_COLLECTION_TEST_CASE.matches(tree, state)) {
+      return Optional.empty();
+    }
+
+    URI sourceFile = state.getPath().getCompilationUnit().getSourceFile().toUri();
+    return Optional.of(
+        RefasterTestCases.create(
+            sourceFile,
+            getRuleCollectionName(tree),
+            isInputFile(sourceFile),
+            getRefasterTestCases(tree, state)));
+  }
+
+  private static String getRuleCollectionName(ClassTree tree) {
+    String className = tree.getSimpleName().toString();
+
+    // XXX: Instead of throwing an error here, it'd be nicer to have a bug checker validate key
+    // aspects of `RefasterRuleCollectionTestCase` subtypes.
+    return tryExtractPatternGroup(className, TEST_CLASS_NAME_PATTERN)
+        .orElseThrow(
+            violation(
+                "Refaster rule collection test class name '%s' does not match '%s'",
+                className, TEST_CLASS_NAME_PATTERN));
+  }
+
+  private static boolean isInputFile(URI sourceFile) {
+    String path = sourceFile.getPath();
+
+    // XXX: Instead of throwing an error here, it'd be nicer to have a bug checker validate key
+    // aspects of `RefasterRuleCollectionTestCase` subtypes.
+    return "Input"
+        .equals(
+            tryExtractPatternGroup(path, TEST_CLASS_FILE_NAME_PATTERN)
+                .orElseThrow(
+                    violation(
+                        "Refaster rule collection test file name '%s' does not match '%s'",
+                        path, TEST_CLASS_FILE_NAME_PATTERN)));
+  }
+
+  private static ImmutableList<RefasterTestCase> getRefasterTestCases(
+      ClassTree tree, VisitorState state) {
+    return tree.getMembers().stream()
+        .filter(MethodTree.class::isInstance)
+        .map(MethodTree.class::cast)
+        .flatMap(m -> tryExtractRefasterTestCase(m, state).stream())
+        .collect(toImmutableList());
+  }
+
+  private static Optional<RefasterTestCase> tryExtractRefasterTestCase(
+      MethodTree method, VisitorState state) {
+    return tryExtractPatternGroup(method.getName().toString(), TEST_METHOD_NAME_PATTERN)
+        .map(name -> RefasterTestCase.create(name, getFormattedSource(method, state)));
+  }
+
+  /**
+   * Returns the source code for the specified method.
+   *
+   * @implNote This operation attempts to trim leading whitespace, such that the start and end of
+   *     the method declaration are aligned. The implemented heuristic assumes that the code is
+   *     formatted using Google Java Format.
+   */
+  // XXX: Leading Javadoc and other comments are currently not extracted. Consider fixing this.
+  private static String getFormattedSource(MethodTree method, VisitorState state) {
+    String source = SourceCode.treeToString(method, state);
+    int finalNewline = source.lastIndexOf(LINE_SEPARATOR);
+    if (finalNewline < 0) {
+      return source;
+    }
+
+    int indentation = Math.max(0, source.lastIndexOf(' ') - finalNewline);
+    String prefixToStrip = " ".repeat(indentation);
+
+    return LINE_SPLITTER
+        .splitToStream(source)
+        .map(line -> line.startsWith(prefixToStrip) ? line.substring(indentation) : line)
+        .collect(joining(LINE_SEPARATOR));
+  }
+
+  private static Optional<String> tryExtractPatternGroup(String input, Pattern pattern) {
+    java.util.regex.Matcher matcher = pattern.matcher(input);
+    return matcher.matches() ? Optional.of(matcher.group(1)) : Optional.empty();
+  }
+
+  @FormatMethod
+  private static Supplier<VerifyException> violation(String format, Object... args) {
+    return () -> new VerifyException(String.format(format, args));
+  }
+
+  @AutoValue
+  @JsonDeserialize(as = AutoValue_RefasterRuleCollectionTestExtractor_RefasterTestCases.class)
+  abstract static class RefasterTestCases {
+    static RefasterTestCases create(
+        URI source,
+        String ruleCollection,
+        boolean isInput,
+        ImmutableList<RefasterTestCase> testCases) {
+      return new AutoValue_RefasterRuleCollectionTestExtractor_RefasterTestCases(
+          source, ruleCollection, isInput, testCases);
+    }
+
+    abstract URI source();
+
+    abstract String ruleCollection();
+
+    abstract boolean isInput();
+
+    abstract ImmutableList<RefasterTestCase> testCases();
+  }
+
+  @AutoValue
+  @JsonDeserialize(as = AutoValue_RefasterRuleCollectionTestExtractor_RefasterTestCase.class)
+  abstract static class RefasterTestCase {
+    static RefasterTestCase create(String name, String content) {
+      return new AutoValue_RefasterRuleCollectionTestExtractor_RefasterTestCase(name, content);
+    }
+
+    abstract String name();
+
+    abstract String content();
+  }
+}

documentation-support/src/test/java/tech/picnic/errorprone/documentation/BugPatternTestExtractorTest.java

@@ -7,10 +7,10 @@ import java.net.URI;
 import java.nio.file.Path;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import tech.picnic.errorprone.documentation.BugPatternTestExtractor.BugPatternTestCase;
+import tech.picnic.errorprone.documentation.BugPatternTestExtractor.BugPatternTestCases;
 import tech.picnic.errorprone.documentation.BugPatternTestExtractor.IdentificationTestEntry;
 import tech.picnic.errorprone.documentation.BugPatternTestExtractor.ReplacementTestEntry;
-import tech.picnic.errorprone.documentation.BugPatternTestExtractor.TestCase;
-import tech.picnic.errorprone.documentation.BugPatternTestExtractor.TestCases;
 
 final class BugPatternTestExtractorTest {
   @Test
@@ -269,11 +269,11 @@ final class BugPatternTestExtractorTest {
     verifyGeneratedFileContent(
         outputDirectory,
         "SingleFileCompilationTestHelperTest",
-        TestCases.create(
+        BugPatternTestCases.create(
             URI.create("file:///SingleFileCompilationTestHelperTest.java"),
             "SingleFileCompilationTestHelperTest",
             ImmutableList.of(
-                TestCase.create(
+                BugPatternTestCase.create(
                     "SingleFileCompilationTestHelperTest.TestChecker",
                     ImmutableList.of(
                         IdentificationTestEntry.create(
@@ -302,11 +302,11 @@ final class BugPatternTestExtractorTest {
     verifyGeneratedFileContent(
         outputDirectory,
         "SingleFileCompilationTestHelperWithSetArgsTest",
-        TestCases.create(
+        BugPatternTestCases.create(
             URI.create("file:///SingleFileCompilationTestHelperWithSetArgsTest.java"),
             "SingleFileCompilationTestHelperWithSetArgsTest",
             ImmutableList.of(
-                TestCase.create(
+                BugPatternTestCase.create(
                     "SingleFileCompilationTestHelperWithSetArgsTest.TestChecker",
                     ImmutableList.of(
                         IdentificationTestEntry.create(
@@ -335,11 +335,11 @@ final class BugPatternTestExtractorTest {
     verifyGeneratedFileContent(
         outputDirectory,
         "MultiFileCompilationTestHelperTest",
-        TestCases.create(
+        BugPatternTestCases.create(
             URI.create("file:///MultiFileCompilationTestHelperTest.java"),
             "MultiFileCompilationTestHelperTest",
             ImmutableList.of(
-                TestCase.create(
+                BugPatternTestCase.create(
                     "MultiFileCompilationTestHelperTest.TestChecker",
                     ImmutableList.of(
                         IdentificationTestEntry.create(
@@ -370,11 +370,11 @@ final class BugPatternTestExtractorTest {
     verifyGeneratedFileContent(
         outputDirectory,
         "SingleFileBugCheckerRefactoringTestHelperTest",
-        TestCases.create(
+        BugPatternTestCases.create(
             URI.create("file:///SingleFileBugCheckerRefactoringTestHelperTest.java"),
             "SingleFileBugCheckerRefactoringTestHelperTest",
             ImmutableList.of(
-                TestCase.create(
+                BugPatternTestCase.create(
                     "SingleFileBugCheckerRefactoringTestHelperTest.TestChecker",
                     ImmutableList.of(
                         ReplacementTestEntry.create(
@@ -408,12 +408,12 @@ final class BugPatternTestExtractorTest {
     verifyGeneratedFileContent(
         outputDirectory,
         "SingleFileBugCheckerRefactoringTestHelperWithSetArgsFixChooserAndCustomTestModeTest",
-        TestCases.create(
+        BugPatternTestCases.create(
             URI.create(
                 "file:///SingleFileBugCheckerRefactoringTestHelperWithSetArgsFixChooserAndCustomTestModeTest.java"),
             "SingleFileBugCheckerRefactoringTestHelperWithSetArgsFixChooserAndCustomTestModeTest",
             ImmutableList.of(
-                TestCase.create(
+                BugPatternTestCase.create(
                     "SingleFileBugCheckerRefactoringTestHelperWithSetArgsFixChooserAndCustomTestModeTest.TestChecker",
                     ImmutableList.of(
                         ReplacementTestEntry.create(
@@ -444,11 +444,11 @@ final class BugPatternTestExtractorTest {
     verifyGeneratedFileContent(
         outputDirectory,
         "MultiFileBugCheckerRefactoringTestHelperTest",
-        TestCases.create(
+        BugPatternTestCases.create(
             URI.create("file:///MultiFileBugCheckerRefactoringTestHelperTest.java"),
             "MultiFileBugCheckerRefactoringTestHelperTest",
             ImmutableList.of(
-                TestCase.create(
+                BugPatternTestCase.create(
                     "MultiFileBugCheckerRefactoringTestHelperTest.TestChecker",
                     ImmutableList.of(
                         ReplacementTestEntry.create(
@@ -484,16 +484,16 @@ final class BugPatternTestExtractorTest {
     verifyGeneratedFileContent(
         outputDirectory,
         "CompilationAndBugCheckerRefactoringTestHelpersTest",
-        TestCases.create(
+        BugPatternTestCases.create(
             URI.create("file:///CompilationAndBugCheckerRefactoringTestHelpersTest.java"),
             "CompilationAndBugCheckerRefactoringTestHelpersTest",
             ImmutableList.of(
-                TestCase.create(
+                BugPatternTestCase.create(
                     "CompilationAndBugCheckerRefactoringTestHelpersTest.TestChecker",
                     ImmutableList.of(
                         IdentificationTestEntry.create(
                             "A.java", "// BUG: Diagnostic contains:\nclass A {}\n"))),
-                TestCase.create(
+                BugPatternTestCase.create(
                     "CompilationAndBugCheckerRefactoringTestHelpersTest.TestChecker",
                     ImmutableList.of(
                         ReplacementTestEntry.create(
@@ -532,17 +532,17 @@ final class BugPatternTestExtractorTest {
     verifyGeneratedFileContent(
         outputDirectory,
         "CompilationAndBugCheckerRefactoringTestHelpersWithCustomCheckerPackageAndNamesTest",
-        TestCases.create(
+        BugPatternTestCases.create(
             URI.create(
                 "file:///CompilationAndBugCheckerRefactoringTestHelpersWithCustomCheckerPackageAndNamesTest.java"),
             "pkg.CompilationAndBugCheckerRefactoringTestHelpersWithCustomCheckerPackageAndNamesTest",
             ImmutableList.of(
-                TestCase.create(
+                BugPatternTestCase.create(
                     "pkg.CompilationAndBugCheckerRefactoringTestHelpersWithCustomCheckerPackageAndNamesTest.CustomTestChecker",
                     ImmutableList.of(
                         IdentificationTestEntry.create(
                             "A.java", "// BUG: Diagnostic contains:\nclass A {}\n"))),
-                TestCase.create(
+                BugPatternTestCase.create(
                     "pkg.CompilationAndBugCheckerRefactoringTestHelpersWithCustomCheckerPackageAndNamesTest.CustomTestChecker2",
                     ImmutableList.of(
                         ReplacementTestEntry.create(
@@ -550,9 +550,9 @@ final class BugPatternTestExtractorTest {
   }
 
   private static void verifyGeneratedFileContent(
-      Path outputDirectory, String testClass, TestCases expected) {
+      Path outputDirectory, String testClass, BugPatternTestCases expected) {
     assertThat(outputDirectory.resolve(String.format("bugpattern-test-%s.json", testClass)))
         .exists()
-        .returns(expected, path -> Json.read(path, TestCases.class));
+        .returns(expected, path -> Json.read(path, BugPatternTestCases.class));
   }
 }

documentation-support/src/test/java/tech/picnic/errorprone/documentation/RefasterRuleCollectionTestExtractorTest.java

@@ -0,0 +1,166 @@
+package tech.picnic.errorprone.documentation;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import com.google.common.base.VerifyException;
+import com.google.common.collect.ImmutableList;
+import java.net.URI;
+import java.nio.file.Path;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import tech.picnic.errorprone.documentation.RefasterRuleCollectionTestExtractor.RefasterTestCase;
+import tech.picnic.errorprone.documentation.RefasterRuleCollectionTestExtractor.RefasterTestCases;
+
+final class RefasterRuleCollectionTestExtractorTest {
+  @Test
+  void noRefasterRuleTest(@TempDir Path outputDirectory) {
+    Compilation.compileWithDocumentationGenerator(
+        outputDirectory, "NoRefasterRuleTest.java", "public final class NoRefasterRuleTest {}");
+
+    assertThat(outputDirectory.toAbsolutePath()).isEmptyDirectory();
+  }
+
+  @Test
+  void invalidTestClassName(@TempDir Path outputDirectory) {
+    assertThatThrownBy(
+            () ->
+                Compilation.compileWithDocumentationGenerator(
+                    outputDirectory,
+                    "InvalidTestClassNameInput.java",
+                    "import tech.picnic.errorprone.refaster.test.RefasterRuleCollectionTestCase;",
+                    "",
+                    "final class InvalidTestClassName implements RefasterRuleCollectionTestCase {}"))
+        .cause()
+        .isInstanceOf(VerifyException.class)
+        .hasMessage(
+            "Refaster rule collection test class name 'InvalidTestClassName' does not match '(.*)Test'");
+  }
+
+  @Test
+  void invalidFileName(@TempDir Path outputDirectory) {
+    assertThatThrownBy(
+            () ->
+                Compilation.compileWithDocumentationGenerator(
+                    outputDirectory,
+                    "InvalidFileNameTest.java",
+                    "import tech.picnic.errorprone.refaster.test.RefasterRuleCollectionTestCase;",
+                    "",
+                    "final class InvalidFileNameTest implements RefasterRuleCollectionTestCase {}"))
+        .cause()
+        .isInstanceOf(VerifyException.class)
+        .hasMessage(
+            "Refaster rule collection test file name '/InvalidFileNameTest.java' does not match '.*(Input|Output)\\.java'");
+  }
+
+  @Test
+  void emptyRefasterRuleCollectionTestInput(@TempDir Path outputDirectory) {
+    Compilation.compileWithDocumentationGenerator(
+        outputDirectory,
+        "EmptyRefasterRuleCollectionTestInput.java",
+        "import tech.picnic.errorprone.refaster.test.RefasterRuleCollectionTestCase;",
+        "",
+        "final class EmptyRefasterRuleCollectionTest implements RefasterRuleCollectionTestCase {}");
+
+    verifyGeneratedFileContent(
+        outputDirectory,
+        "EmptyRefasterRuleCollectionTestInput",
+        RefasterTestCases.create(
+            URI.create("file:///EmptyRefasterRuleCollectionTestInput.java"),
+            "EmptyRefasterRuleCollection",
+            /* isInput= */ true,
+            ImmutableList.of()));
+  }
+
+  @Test
+  void singletonRefasterRuleCollectionTestOutput(@TempDir Path outputDirectory) {
+    Compilation.compileWithDocumentationGenerator(
+        outputDirectory,
+        "SingletonRefasterRuleCollectionTestOutput.java",
+        "import tech.picnic.errorprone.refaster.test.RefasterRuleCollectionTestCase;",
+        "",
+        "final class SingletonRefasterRuleCollectionTest implements RefasterRuleCollectionTestCase {",
+        "  int testMyRule() {",
+        "    return 42;",
+        "  }",
+        "}");
+
+    verifyGeneratedFileContent(
+        outputDirectory,
+        "SingletonRefasterRuleCollectionTestOutput",
+        RefasterTestCases.create(
+            URI.create("file:///SingletonRefasterRuleCollectionTestOutput.java"),
+            "SingletonRefasterRuleCollection",
+            /* isInput= */ false,
+            ImmutableList.of(
+                RefasterTestCase.create(
+                    "MyRule",
+                    """
+                    int testMyRule() {
+                      return 42;
+                    }"""))));
+  }
+
+  @Test
+  void complexRefasterRuleCollectionTestOutput(@TempDir Path outputDirectory) {
+    Compilation.compileWithDocumentationGenerator(
+        outputDirectory,
+        "pkg/ComplexRefasterRuleCollectionTestInput.java",
+        "package pkg;",
+        "",
+        "import com.google.common.collect.ImmutableSet;",
+        "import tech.picnic.errorprone.refaster.test.RefasterRuleCollectionTestCase;",
+        "",
+        "final class ComplexRefasterRuleCollectionTest implements RefasterRuleCollectionTestCase {",
+        "  private static final String IGNORED_CONSTANT = \"constant\";",
+        "",
+        "  @Override",
+        "  public ImmutableSet<Object> elidedTypesAndStaticImports() {",
+        "    return ImmutableSet.of();",
+        "  }",
+        "",
+        "  /** Javadoc. */",
+        "  String testFirstRule() {",
+        "    return \"Don't panic\";",
+        "  }",
+        "",
+        "  // Comment.",
+        "  String testSecondRule() {",
+        "    return \"Carry a towel\";",
+        "  }",
+        "",
+        "  void testEmptyRule() {}",
+        "}");
+
+    verifyGeneratedFileContent(
+        outputDirectory,
+        "ComplexRefasterRuleCollectionTestInput",
+        RefasterTestCases.create(
+            URI.create("file:///pkg/ComplexRefasterRuleCollectionTestInput.java"),
+            "ComplexRefasterRuleCollection",
+            /* isInput= */ true,
+            ImmutableList.of(
+                RefasterTestCase.create(
+                    "FirstRule",
+                    """
+                    String testFirstRule() {
+                      return "Don't panic";
+                    }"""),
+                RefasterTestCase.create(
+                    "SecondRule",
+                    """
+                    String testSecondRule() {
+                      return "Carry a towel";
+                    }"""),
+                RefasterTestCase.create("EmptyRule", "void testEmptyRule() {}"))));
+  }
+
+  private static void verifyGeneratedFileContent(
+      Path outputDirectory, String testIdentifier, RefasterTestCases expected) {
+    assertThat(
+            outputDirectory.resolve(
+                String.format("refaster-rule-collection-test-%s.json", testIdentifier)))
+        .exists()
+        .returns(expected, path -> Json.read(path, RefasterTestCases.class));
+  }
+}