Optimize code, introduce benchmark, simplify test

pom.xml

@@ -153,9 +153,13 @@
         <version.error-prone-slf4j>0.1.15</version.error-prone-slf4j>
         <version.guava-beta-checker>1.0</version.guava-beta-checker>
         <version.jdk>11</version.jdk>
+        <version.jmh>1.35</version.jmh>
         <version.maven>3.8.6</version.maven>
         <version.mockito>4.8.0</version.mockito>
-        <version.nopen-checker>1.0.1</version.nopen-checker>
+        <!-- XXX: Consider providing our own implementation with similar
+        functionaliy, and designing it such that JMH classes would not need to
+        be annotated `@Open`. -->
+        <version.nopen>1.0.1</version.nopen>
         <version.nullaway>0.10.2</version.nullaway>
         <!-- XXX: Two other dependencies are potentially of interest:
         `com.palantir.assertj-automation:assertj-refaster-rules` and
@@ -272,12 +276,10 @@
                 <artifactId>truth</artifactId>
                 <version>1.1.3</version>
             </dependency>
-            <!-- Specified as a workaround for
-            https://github.com/mojohaus/versions-maven-plugin/issues/244. -->
             <dependency>
                 <groupId>com.jakewharton.nopen</groupId>
-                <artifactId>nopen-checker</artifactId>
-                <version>${version.nopen-checker}</version>
+                <artifactId>nopen-annotations</artifactId>
+                <version>${version.nopen}</version>
             </dependency>
             <dependency>
                 <groupId>com.newrelic.agent.java</groupId>
@@ -401,6 +403,11 @@
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
+            <dependency>
+                <groupId>org.openjdk.jmh</groupId>
+                <artifactId>jmh-core</artifactId>
+                <version>${version.jmh}</version>
+            </dependency>
             <dependency>
                 <groupId>org.slf4j</groupId>
                 <artifactId>slf4j-api</artifactId>
@@ -846,7 +853,7 @@
                             <path>
                                 <groupId>com.jakewharton.nopen</groupId>
                                 <artifactId>nopen-checker</artifactId>
-                                <version>${version.nopen-checker}</version>
+                                <version>${version.nopen}</version>
                             </path>
                             <!-- XXX: Before enabling these plugins we'll
                             need to resolve some violations. Some of the
@@ -877,6 +884,11 @@
                                 <artifactId>mockito-errorprone</artifactId>
                                 <version>${version.mockito}</version>
                             </path>
+                            <path>
+                                <groupId>org.openjdk.jmh</groupId>
+                                <artifactId>jmh-generator-annprocess</artifactId>
+                                <version>${version.jmh}</version>
+                            </path>
                         </annotationProcessorPaths>
                         <compilerArgs>
                             <arg>--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED</arg>
@@ -1119,6 +1131,11 @@
                     <artifactId>build-helper-maven-plugin</artifactId>
                     <version>3.3.0</version>
                 </plugin>
+                <plugin>
+                    <groupId>org.codehaus.mojo</groupId>
+                    <artifactId>exec-maven-plugin</artifactId>
+                    <version>3.1.0</version>
+                </plugin>
                 <plugin>
                     <groupId>org.codehaus.mojo</groupId>
                     <artifactId>license-maven-plugin</artifactId>
@@ -1193,6 +1210,7 @@
                                 <!-- -->
                                 GPL-2.0-with-classpath-exception
                                 | CDDL/GPLv2+CE
+                                | GNU General Public License (GPL), version 2, with the Classpath exception
                                 | GNU General Public License, version 2 (GPL2), with the classpath exception
                                 | GNU General Public License, version 2, with the Classpath Exception
                                 | GPL2 w/ CPE
@@ -1558,6 +1576,7 @@
                                     avoid that, so we simply tell Error Prone
                                     not to warn about generated code. -->
                                     -XepDisableWarningsInGeneratedCode
+                                    -XepExcludedPaths:\Q${project.build.directory}${file.separator}\E.*
                                     <!-- We don't target Android. -->
                                     -Xep:AndroidJdkLibsChecker:OFF
                                     <!-- XXX: Enable this once we open-source
@@ -1771,5 +1790,66 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <!-- Enables execution of a JMH benchmark. Given a benchmark class
+            `tech.picnic.MyBenchmark`, the following command (executed against
+            the (sub)module in which the benchmark resides) will compile and
+            execute said benchmark:
+
+              mvn process-test-classes -Dverification.skip \
+                -Djmh.run-benchmark=tech.picnic.MyBenchmark
+            -->
+            <id>run-jmh-benchmark</id>
+            <activation>
+                <property>
+                    <name>jmh.run-benchmark</name>
+                </property>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-dependency-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>build-jmh-runtime-classpath</id>
+                                <goals>
+                                    <goal>build-classpath</goal>
+                                </goals>
+                                <configuration>
+                                    <outputProperty>testClasspath</outputProperty>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>run-jmh-benchmark</id>
+                                <goals>
+                                    <goal>java</goal>
+                                </goals>
+                                <phase>process-test-classes</phase>
+                                <configuration>
+                                    <classpathScope>test</classpathScope>
+                                    <mainClass>${jmh.run-benchmark}</mainClass>
+                                    <systemProperties>
+                                        <!-- The runtime classpath is defined
+                                        in this way so that any JVMs forked by
+                                        JMH will have the desired classpath. -->
+                                        <systemProperty>
+                                            <key>java.class.path</key>
+                                            <value>${project.build.testOutputDirectory}${path.separator}${project.build.outputDirectory}${path.separator}${testClasspath}</value>
+                                        </systemProperty>
+                                    </systemProperties>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
     </profiles>
 </project>

refaster-runner/pom.xml

@@ -66,6 +66,11 @@
             <artifactId>guava</artifactId>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.jakewharton.nopen</groupId>
+            <artifactId>nopen-annotations</artifactId>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>org.assertj</groupId>
             <artifactId>assertj-core</artifactId>
@@ -86,6 +91,11 @@
             <artifactId>junit-jupiter-params</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-core</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

refaster-runner/src/main/java/tech/picnic/errorprone/refaster/runner/Node.java

@@ -8,6 +8,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Maps;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -35,6 +36,8 @@ abstract class Node<T> {
 
   // XXX: Consider having `RefasterRuleSelector` already collect the candidate edges into a
   // `SortedSet`, as that would likely speed up `ImmutableSortedSet#copyOf`.
+  // XXX: If this ^ proves worthwhile, then the test code and benchmark should be updated
+  // accordingly.
   void collectReachableValues(Set<String> candidateEdges, Consumer<T> sink) {
     collectReachableValues(ImmutableSortedSet.copyOf(candidateEdges).asList(), sink);
   }
@@ -86,9 +89,9 @@ abstract class Node<T> {
     private void register(
         List<T> values, Function<? super T, ? extends Set<? extends Set<String>>> pathsExtractor) {
       for (T value : values) {
-        pathsExtractor.apply(value).stream()
-            .sorted(comparingInt(Set::size))
-            .forEach(path -> registerPath(value, ImmutableList.sortedCopyOf(path)));
+        List<? extends Set<String>> paths = new ArrayList<>(pathsExtractor.apply(value));
+        Collections.sort(paths, comparingInt(Set::size));
+        paths.forEach(path -> registerPath(value, ImmutableList.sortedCopyOf(path)));
       }
     }
 
@@ -98,14 +101,13 @@ abstract class Node<T> {
         return;
       }
 
-      path.stream()
-          .findFirst()
-          .ifPresentOrElse(
-              edge ->
-                  children()
-                      .computeIfAbsent(edge, k -> create())
-                      .registerPath(value, path.subList(1, path.size())),
-              () -> values().add(value));
+      if (path.isEmpty()) {
+        values().add(value);
+      } else {
+        children()
+            .computeIfAbsent(path.get(0), k -> create())
+            .registerPath(value, path.subList(1, path.size()));
+      }
     }
 
     private Node<T> immutable() {

refaster-runner/src/main/java/tech/picnic/errorprone/refaster/runner/Refaster.java

@@ -157,6 +157,8 @@ public final class Refaster extends BugChecker implements CompilationUnitTreeMat
     return description.fixes.stream().flatMap(fix -> fix.getReplacements(endPositions).stream());
   }
 
+  // XXX: Add a flag to disable the optimized `RefasterRuleSelector`. That would allow us to verify
+  // that we're not prematurely pruning rules.
   private static RefasterRuleSelector createRefasterRuleSelector(ErrorProneFlags flags) {
     ImmutableListMultimap<String, CodeTransformer> allTransformers =
         CodeTransformers.getAllCodeTransformers();

refaster-runner/src/main/java/tech/picnic/errorprone/refaster/runner/RefasterRuleSelector.java

@@ -19,14 +19,18 @@ import com.google.errorprone.refaster.UStaticIdent;
 import com.google.errorprone.refaster.annotation.BeforeTemplate;
 import com.sun.source.tree.AssignmentTree;
 import com.sun.source.tree.BinaryTree;
+import com.sun.source.tree.ClassTree;
 import com.sun.source.tree.CompilationUnitTree;
 import com.sun.source.tree.CompoundAssignmentTree;
 import com.sun.source.tree.ExpressionTree;
 import com.sun.source.tree.IdentifierTree;
 import com.sun.source.tree.MemberReferenceTree;
 import com.sun.source.tree.MemberSelectTree;
+import com.sun.source.tree.MethodTree;
+import com.sun.source.tree.PackageTree;
 import com.sun.source.tree.Tree;
 import com.sun.source.tree.UnaryTree;
+import com.sun.source.tree.VariableTree;
 import com.sun.source.util.TreeScanner;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
@@ -183,7 +187,7 @@ final class RefasterRuleSelector {
    * @throws IllegalArgumentException If the given input is not supported.
    */
   // XXX: Extend list to cover remaining cases; at least for any `Kind` that may appear in a
-  // Refaster template.
+  // Refaster template. (E.g. keywords such as `if`, `instanceof`, `new`, ...)
   private static String treeKindToString(Tree.Kind kind) {
     switch (kind) {
       case ASSIGNMENT:
@@ -458,6 +462,40 @@ final class RefasterRuleSelector {
   }
 
   private static class SourceIdentifierExtractor extends TreeScanner<Void, Set<String>> {
+    @Nullable
+    @Override
+    public Void visitPackage(PackageTree node, Set<String> identifiers) {
+      /* Refaster rules never match package declarations. */
+      return null;
+    }
+
+    @Nullable
+    @Override
+    public Void visitClass(ClassTree node, Set<String> identifiers) {
+      /*
+       * Syntactic details of a class declaration other than the definition of its members do not
+       * need to be reflected in a Refaster rule for it to apply to the class's code.
+       */
+      return scan(node.getMembers(), identifiers);
+    }
+
+    @Nullable
+    @Override
+    public Void visitMethod(MethodTree node, Set<String> identifiers) {
+      /*
+       * Syntactic details of a method declaration other than its body do not need to be reflected
+       * in a Refaster rule for it to apply to the method's code.
+       */
+      return scan(node.getBody(), identifiers);
+    }
+
+    @Nullable
+    @Override
+    public Void visitVariable(VariableTree node, Set<String> identifiers) {
+      /* A variable's modifiers and name do not influence where a Refaster rule matches. */
+      return reduce(scan(node.getInitializer(), identifiers), scan(node.getType(), identifiers));
+    }
+
     @Nullable
     @Override
     public Void visitIdentifier(IdentifierTree node, Set<String> identifiers) {

refaster-runner/src/test/java/tech/picnic/errorprone/refaster/runner/NodeBenchmark.java

@@ -0,0 +1,81 @@
+package tech.picnic.errorprone.refaster.runner;
+
+import static com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap;
+import static java.util.function.Function.identity;
+
+import com.google.common.collect.ImmutableListMultimap;
+import com.jakewharton.nopen.annotation.Open;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import tech.picnic.errorprone.refaster.runner.NodeTestCase.NodeTestCaseEntry;
+
+@Open
+@State(Scope.Benchmark)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Fork(jvmArgs = {"-Xms1G", "-Xmx1G"})
+@Warmup(iterations = 5)
+@Measurement(iterations = 10)
+public class NodeBenchmark {
+  @SuppressWarnings("NullAway" /* Initialized by `@Setup` method. */)
+  private ImmutableListMultimap<NodeTestCase<Integer>, NodeTestCaseEntry<Integer>> testCases;
+
+  public static void main(String[] args) throws RunnerException {
+    String testRegex = Pattern.quote(NodeBenchmark.class.getCanonicalName());
+    new Runner(new OptionsBuilder().include(testRegex).forks(1).build()).run();
+  }
+
+  @Setup
+  public final void setUp() {
+    Random random = new Random(0);
+
+    testCases =
+        Stream.of(
+                NodeTestCase.generate(100, 5, 10, 10, random),
+                NodeTestCase.generate(100, 5, 10, 100, random),
+                NodeTestCase.generate(100, 5, 10, 1000, random),
+                NodeTestCase.generate(1000, 10, 20, 10, random),
+                NodeTestCase.generate(1000, 10, 20, 100, random),
+                NodeTestCase.generate(1000, 10, 20, 1000, random),
+                NodeTestCase.generate(1000, 10, 20, 10000, random))
+            .collect(
+                flatteningToImmutableListMultimap(
+                    identity(), testCase -> testCase.generateTestCaseEntries(random)));
+  }
+
+  @Benchmark
+  public final void create(Blackhole bh) {
+    for (NodeTestCase<Integer> testCase : testCases.keySet()) {
+      bh.consume(testCase.buildTree());
+    }
+  }
+
+  @Benchmark
+  public final void collectReachableValues(Blackhole bh) {
+    for (Map.Entry<NodeTestCase<Integer>, Collection<NodeTestCaseEntry<Integer>>> e :
+        testCases.asMap().entrySet()) {
+      Node<Integer> tree = e.getKey().buildTree();
+      for (NodeTestCaseEntry<Integer> testCaseEntry : e.getValue()) {
+        tree.collectReachableValues(testCaseEntry.candidateEdges(), bh::consume);
+      }
+    }
+  }
+}

refaster-runner/src/test/java/tech/picnic/errorprone/refaster/runner/NodeTest.java

@@ -1,176 +1,47 @@
 package tech.picnic.errorprone.refaster.runner;
 
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static com.google.common.collect.ImmutableSetMultimap.flatteningToImmutableSetMultimap;
-import static java.util.function.Function.identity;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.params.provider.Arguments.arguments;
 
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSetMultimap;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
+import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 import java.util.Random;
-import java.util.Set;
 import java.util.stream.Stream;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 
 final class NodeTest {
-  private static Stream<Arguments> verifyTestCases() {
+  private static Stream<Arguments> collectReachableValuesTestCases() {
     Random random = new Random(0);
 
-    /* { treeInput, random } */
     return Stream.of(
-        arguments(generateTestInput(random, 0, 0, 0, 0), random),
-        arguments(generateTestInput(random, 1, 1, 1, 1), random),
-        arguments(generateTestInput(random, 2, 2, 2, 10), random),
-        arguments(generateTestInput(random, 2, 2, 2, 100), random),
-        arguments(generateTestInput(random, 2, 2, 5, 10), random),
-        arguments(generateTestInput(random, 2, 2, 5, 100), random),
-        arguments(generateTestInput(random, 2, 2, 10, 10), random),
-        arguments(generateTestInput(random, 2, 2, 10, 100), random),
-        arguments(generateTestInput(random, 100, 1, 1, 10), random),
-        arguments(generateTestInput(random, 100, 1, 1, 100), random),
-        arguments(generateTestInput(random, 100, 1, 5, 10), random),
-        arguments(generateTestInput(random, 100, 1, 5, 100), random),
-        arguments(generateTestInput(random, 100, 5, 5, 10), random),
-        arguments(generateTestInput(random, 100, 5, 5, 100), random),
-        arguments(generateTestInput(random, 100, 5, 5, 1000), random),
-        arguments(generateTestInput(random, 100, 5, 10, 10), random),
-        arguments(generateTestInput(random, 100, 5, 10, 100), random),
-        arguments(generateTestInput(random, 100, 5, 10, 1000), random),
-        arguments(generateTestInput(random, 1000, 1, 5, 10), random),
-        arguments(generateTestInput(random, 1000, 1, 5, 100), random),
-        arguments(generateTestInput(random, 1000, 1, 5, 1000), random),
-        arguments(generateTestInput(random, 1000, 5, 5, 10), random),
-        arguments(generateTestInput(random, 1000, 5, 5, 100), random),
-        arguments(generateTestInput(random, 1000, 5, 5, 1000), random),
-        arguments(generateTestInput(random, 1000, 10, 5, 10), random),
-        arguments(generateTestInput(random, 1000, 10, 5, 100), random),
-        arguments(generateTestInput(random, 1000, 10, 5, 1000), random),
-        arguments(generateTestInput(random, 1000, 10, 5, 10000), random),
-        arguments(generateTestInput(random, 1000, 5, 10, 10), random),
-        arguments(generateTestInput(random, 1000, 5, 10, 100), random),
-        arguments(generateTestInput(random, 1000, 5, 10, 1000), random),
-        arguments(generateTestInput(random, 1000, 5, 10, 10000), random));
+            NodeTestCase.generate(0, 0, 0, 0, random),
+            NodeTestCase.generate(1, 1, 1, 1, random),
+            NodeTestCase.generate(2, 2, 2, 10, random),
+            NodeTestCase.generate(10, 2, 5, 10, random),
+            NodeTestCase.generate(10, 2, 5, 100, random),
+            NodeTestCase.generate(100, 5, 10, 100, random),
+            NodeTestCase.generate(100, 5, 10, 1000, random))
+        .flatMap(
+            testCase -> {
+              Node<Integer> tree = testCase.buildTree();
+              return testCase
+                  .generateTestCaseEntries(random)
+                  .map(e -> arguments(tree, e.candidateEdges(), e.reachableValues()));
+            });
   }
 
-  @MethodSource("verifyTestCases")
+  @MethodSource("collectReachableValuesTestCases")
   @ParameterizedTest
-  void verify(ImmutableSetMultimap<Integer, ImmutableSet<String>> treeInput, Random random) {
-    Node<Integer> tree = Node.create(treeInput.keySet().asList(), treeInput::get);
-
-    // XXX: Drop.
-    // System.out.println(size(tree));
-
-    verifyConstruction(tree, treeInput, random);
-  }
-
-  // XXX: Drop.
-  // private static <T> int size(Node<T> t) {
-  //  return t.values().size()
-  //      + t.children().size()
-  //      + t.children().values().stream().mapToInt(NodeTest::size).sum();
-  // }
-
-  private static void verifyConstruction(
+  void collectReachableValues(
       Node<Integer> tree,
-      ImmutableSetMultimap<Integer, ImmutableSet<String>> treeInput,
-      Random random) {
-    ImmutableSet<String> allPathValues =
-        treeInput.values().stream().flatMap(ImmutableSet::stream).collect(toImmutableSet());
-
-    for (Map.Entry<Integer, ImmutableSet<String>> e : treeInput.entries()) {
-      verifyReachability(tree, e.getKey(), shuffle(e.getValue(), random), allPathValues, random);
-    }
-  }
-
-  private static <T> void verifyReachability(
-      Node<T> tree,
-      T leaf,
-      ImmutableSet<String> unorderedEdgesToLeaf,
-      ImmutableSet<String> allEdges,
-      Random random) {
-    String unknownEdge = "unknown";
-
-    assertThat(isReachable(tree, leaf, unorderedEdgesToLeaf)).isTrue();
-    assertThat(isReachable(tree, leaf, insertValue(unorderedEdgesToLeaf, unknownEdge, random)))
-        .isTrue();
-    if (!allEdges.isEmpty()) {
-      String knownEdge = selectRandomElement(allEdges, random);
-      assertThat(isReachable(tree, leaf, insertValue(unorderedEdgesToLeaf, knownEdge, random)))
-          .isTrue();
-    }
-
-    // XXX: Strictly speaking this is wrong: these paths _could_ exist.
-    // XXX: Implement something better.
-    if (!unorderedEdgesToLeaf.isEmpty()) {
-      // assertThat(isReachable(tree, leaf, randomStrictSubset(unorderedEdgesToLeaf, random)))
-      //    .isFalse();
-      // assertThat(
-      //        isReachable(
-      //            tree,
-      //            leaf,
-      //            insertValue(
-      //                randomStrictSubset(unorderedEdgesToLeaf, random), unknownEdge, random)))
-      //    .isFalse();
-    }
-  }
-
-  private static <T> ImmutableSet<T> shuffle(ImmutableSet<T> values, Random random) {
-    List<T> allValues = new ArrayList<>(values);
-    Collections.shuffle(allValues, random);
-    return ImmutableSet.copyOf(allValues);
-  }
-
-  private static <T> ImmutableSet<T> insertValue(
-      ImmutableSet<T> values, T extraValue, Random random) {
-    List<T> allValues = new ArrayList<>(values);
-    allValues.add(random.nextInt(values.size() + 1), extraValue);
-    return ImmutableSet.copyOf(allValues);
-  }
-
-  private static <T> T selectRandomElement(ImmutableSet<T> collection, Random random) {
-    return collection.asList().get(random.nextInt(collection.size()));
-  }
-
-  // XXX: Use or drop.
-  // private static <T> ImmutableSet<T> randomStrictSubset(ImmutableSet<T> values, Random random) {
-  //  checkArgument(!values.isEmpty(), "Cannot select strict subset of random collection");
-  //
-  //  List<T> allValues = new ArrayList<>(values);
-  //  Collections.shuffle(allValues, random);
-  //  return ImmutableSet.copyOf(allValues.subList(0, random.nextInt(allValues.size())));
-  // }
-
-  private static <T> boolean isReachable(
-      Node<T> tree, T target, ImmutableSet<String> candidateEdges) {
-    Set<T> matches = new HashSet<>();
-    tree.collectReachableValues(candidateEdges, matches::add);
-    return matches.contains(target);
-  }
-
-  private static ImmutableSetMultimap<Integer, ImmutableSet<String>> generateTestInput(
-      Random random, int entryCount, int maxPathCount, int maxPathLength, int pathValueDomainSize) {
-    return random
-        .ints(entryCount)
-        .boxed()
-        .collect(
-            flatteningToImmutableSetMultimap(
-                identity(),
-                i ->
-                    random
-                        .ints(random.nextInt(maxPathCount + 1))
-                        .mapToObj(
-                            p ->
-                                random
-                                    .ints(random.nextInt(maxPathLength + 1), 0, pathValueDomainSize)
-                                    .mapToObj(String::valueOf)
-                                    .collect(toImmutableSet()))));
+      ImmutableSet<String> candidateEdges,
+      Collection<Integer> expectedReachable) {
+    List<Integer> actualReachable = new ArrayList<>();
+    tree.collectReachableValues(candidateEdges, actualReachable::add);
+    assertThat(actualReachable).hasSameElementsAs(expectedReachable);
   }
 }

refaster-runner/src/test/java/tech/picnic/errorprone/refaster/runner/NodeTestCase.java

@@ -0,0 +1,138 @@
+package tech.picnic.errorprone.refaster.runner;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static com.google.common.collect.ImmutableSetMultimap.flatteningToImmutableSetMultimap;
+import static java.util.function.Function.identity;
+import static java.util.stream.Collectors.collectingAndThen;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Random;
+import java.util.stream.Stream;
+
+@AutoValue
+abstract class NodeTestCase<V> {
+  static NodeTestCase<Integer> generate(
+      int entryCount, int maxPathCount, int maxPathLength, int pathValueDomainSize, Random random) {
+    return random
+        .ints(entryCount)
+        .boxed()
+        .collect(
+            collectingAndThen(
+                flatteningToImmutableSetMultimap(
+                    identity(),
+                    i ->
+                        random
+                            .ints(random.nextInt(maxPathCount + 1))
+                            .mapToObj(
+                                p ->
+                                    random
+                                        .ints(
+                                            random.nextInt(maxPathLength + 1),
+                                            0,
+                                            pathValueDomainSize)
+                                        .mapToObj(String::valueOf)
+                                        .collect(toImmutableSet()))),
+                AutoValue_NodeTestCase::new));
+  }
+
+  abstract ImmutableSetMultimap<V, ImmutableSet<String>> input();
+
+  final Node<V> buildTree() {
+    return Node.create(input().keySet().asList(), input()::get);
+  }
+
+  final Stream<NodeTestCaseEntry<V>> generateTestCaseEntries(Random random) {
+    return generatePathTestCases(input(), random);
+  }
+
+  private static <V> Stream<NodeTestCaseEntry<V>> generatePathTestCases(
+      ImmutableSetMultimap<V, ImmutableSet<String>> treeInput, Random random) {
+    ImmutableSet<String> allEdges =
+        treeInput.values().stream().flatMap(ImmutableSet::stream).collect(toImmutableSet());
+
+    return Stream.concat(
+            Stream.of(ImmutableSet.<String>of()), shuffle(treeInput.values(), random).stream())
+        .limit(random.nextInt(20, 100))
+        .flatMap(edges -> generateVariations(edges, allEdges, "unused", random))
+        .distinct()
+        .map(edges -> createTestCaseEntry(treeInput, edges));
+  }
+
+  private static <T> Stream<ImmutableSet<T>> generateVariations(
+      ImmutableSet<T> baseEdges, ImmutableSet<T> allEdges, T unusedEdge, Random random) {
+    Optional<T> knownEdge = selectRandomElement(allEdges, random);
+
+    return Stream.of(
+            random.nextBoolean() ? null : baseEdges,
+            random.nextBoolean() ? null : shuffle(baseEdges, random),
+            random.nextBoolean() ? null : insertValue(baseEdges, unusedEdge, random),
+            baseEdges.isEmpty() || random.nextBoolean()
+                ? null
+                : randomStrictSubset(baseEdges, random),
+            baseEdges.isEmpty() || random.nextBoolean()
+                ? null
+                : insertValue(randomStrictSubset(baseEdges, random), unusedEdge, random),
+            baseEdges.isEmpty() || random.nextBoolean()
+                ? null
+                : knownEdge
+                    .map(edge -> insertValue(randomStrictSubset(baseEdges, random), edge, random))
+                    .orElse(null))
+        .filter(Objects::nonNull);
+  }
+
+  private static <T> Optional<T> selectRandomElement(ImmutableSet<T> collection, Random random) {
+    return collection.isEmpty()
+        ? Optional.empty()
+        : Optional.of(collection.asList().get(random.nextInt(collection.size())));
+  }
+
+  private static <T> ImmutableSet<T> shuffle(ImmutableCollection<T> values, Random random) {
+    List<T> allValues = new ArrayList<>(values);
+    Collections.shuffle(allValues, random);
+    return ImmutableSet.copyOf(allValues);
+  }
+
+  private static <T> ImmutableSet<T> insertValue(
+      ImmutableSet<T> values, T extraValue, Random random) {
+    List<T> allValues = new ArrayList<>(values);
+    allValues.add(random.nextInt(values.size() + 1), extraValue);
+    return ImmutableSet.copyOf(allValues);
+  }
+
+  private static <T> ImmutableSet<T> randomStrictSubset(ImmutableSet<T> values, Random random) {
+    checkArgument(!values.isEmpty(), "Cannot select strict subset of random collection");
+
+    List<T> allValues = new ArrayList<>(values);
+    Collections.shuffle(allValues, random);
+    return ImmutableSet.copyOf(allValues.subList(0, random.nextInt(allValues.size())));
+  }
+
+  private static <V> NodeTestCaseEntry<V> createTestCaseEntry(
+      ImmutableSetMultimap<V, ImmutableSet<String>> treeInput, ImmutableSet<String> edges) {
+    return new AutoValue_NodeTestCase_NodeTestCaseEntry<>(
+        edges,
+        treeInput.asMap().entrySet().stream()
+            .filter(e -> e.getValue().stream().anyMatch(edges::containsAll))
+            .map(Map.Entry::getKey)
+            .collect(toImmutableList()));
+  }
+
+  @AutoValue
+  abstract static class NodeTestCaseEntry<V> {
+    abstract ImmutableSet<String> candidateEdges();
+
+    abstract ImmutableList<V> reachableValues();
+  }
+}