WIP: Benchmark generation

documentation-support/src/main/java/tech/picnic/errorprone/documentation/RefasterBenchmarkGenerator.java

@@ -0,0 +1,56 @@
+package tech.picnic.errorprone.documentation;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.service.AutoService;
+import com.google.common.annotations.VisibleForTesting;
+import com.sun.source.util.JavacTask;
+import com.sun.source.util.Plugin;
+import com.sun.tools.javac.api.BasicJavacTask;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/** A compiler {@link Plugin} that generates JMH benchmarks for Refaster rules. */
+// XXX: Move logic to separate package and Maven module.
+// XXX: Review whether this can be an annotation processor instead.
+@AutoService(Plugin.class)
+public final class RefasterBenchmarkGenerator implements Plugin {
+  @VisibleForTesting static final String OUTPUT_DIRECTORY_FLAG = "-XoutputDirectory";
+  private static final Pattern OUTPUT_DIRECTORY_FLAG_PATTERN =
+      Pattern.compile(Pattern.quote(OUTPUT_DIRECTORY_FLAG) + "=(.*)");
+
+  /** Instantiates a new {@link RefasterBenchmarkGenerator} instance. */
+  public RefasterBenchmarkGenerator() {}
+
+  @Override
+  public String getName() {
+    return getClass().getSimpleName();
+  }
+
+  @Override
+  public void init(JavacTask javacTask, String... args) {
+    checkArgument(args.length == 1, "Precisely one path must be provided");
+
+    // XXX: Drop all path logic: instead generate in the same package as the Refaster rule
+    // collection.
+    javacTask.addTaskListener(
+        new RefasterBenchmarkGeneratorTaskListener(
+            ((BasicJavacTask) javacTask).getContext(), getOutputPath(args[0])));
+  }
+
+  @VisibleForTesting
+  static Path getOutputPath(String pathArg) {
+    Matcher matcher = OUTPUT_DIRECTORY_FLAG_PATTERN.matcher(pathArg);
+    checkArgument(
+        matcher.matches(), "'%s' must be of the form '%s=<value>'", pathArg, OUTPUT_DIRECTORY_FLAG);
+
+    String path = matcher.group(1);
+    try {
+      return Path.of(path);
+    } catch (InvalidPathException e) {
+      throw new IllegalArgumentException(String.format("Invalid path '%s'", path), e);
+    }
+  }
+}

documentation-support/src/main/java/tech/picnic/errorprone/documentation/RefasterBenchmarkGeneratorTaskListener.java

@@ -0,0 +1,94 @@
+package tech.picnic.errorprone.documentation;
+
+import com.google.errorprone.VisitorState;
+import com.google.errorprone.matchers.Matcher;
+import com.google.errorprone.matchers.Matchers;
+import com.sun.source.tree.ClassTree;
+import com.sun.source.tree.CompilationUnitTree;
+import com.sun.source.tree.Tree;
+import com.sun.source.util.TaskEvent;
+import com.sun.source.util.TaskEvent.Kind;
+import com.sun.source.util.TaskListener;
+import com.sun.source.util.TreePath;
+import com.sun.source.util.TreePathScanner;
+import com.sun.tools.javac.api.JavacTrees;
+import com.sun.tools.javac.util.Context;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import javax.tools.JavaFileObject;
+import org.jspecify.annotations.Nullable;
+
+// XXX: Document.
+final class RefasterBenchmarkGeneratorTaskListener implements TaskListener {
+  private final Context context;
+  private final Path outputPath;
+
+  RefasterBenchmarkGeneratorTaskListener(Context context, Path outputPath) {
+    this.context = context;
+    this.outputPath = outputPath;
+  }
+
+  @Override
+  public void started(TaskEvent taskEvent) {
+    if (taskEvent.getKind() == Kind.ANALYZE) {
+      createOutputDirectory();
+    }
+  }
+
+  @Override
+  public void finished(TaskEvent taskEvent) {
+    if (taskEvent.getKind() != Kind.ANALYZE) {
+      return;
+    }
+
+    JavaFileObject sourceFile = taskEvent.getSourceFile();
+    CompilationUnitTree compilationUnit = taskEvent.getCompilationUnit();
+    ClassTree classTree = JavacTrees.instance(context).getTree(taskEvent.getTypeElement());
+    if (sourceFile == null || compilationUnit == null || classTree == null) {
+      return;
+    }
+
+    VisitorState state =
+        VisitorState.createForUtilityPurposes(context)
+            .withPath(new TreePath(new TreePath(compilationUnit), classTree));
+
+    // XXX: Make static.
+    Matcher<Tree> isBenchmarked =
+        Matchers.hasAnnotation("tech.picnic.errorprone.refaster.annotation.Benchmarked");
+
+    new TreePathScanner<@Nullable Void, Boolean>() {
+      @Override
+      public @Nullable Void visitClass(ClassTree classTree, Boolean doBenchmark) {
+        // XXX: Validate that `@Benchmarked` is only placed in contexts with at least one Refaster
+        // rule.
+        boolean inspectClass = doBenchmark || isBenchmarked.matches(classTree, state);
+
+        if (inspectClass) {
+          // XXX: If this class has a `@BeforeTemplate` method, generate a benchmark for it.
+        }
+
+        return super.visitClass(classTree, inspectClass);
+      }
+    }.scan(compilationUnit, false);
+  }
+
+  private void createOutputDirectory() {
+    try {
+      Files.createDirectories(outputPath);
+    } catch (IOException e) {
+      throw new IllegalStateException(
+          String.format("Error while creating directory with path '%s'", outputPath), e);
+    }
+  }
+
+  private <T> void writeToFile(String identifier, String className, T data) {
+    Json.write(outputPath.resolve(String.format("%s-%s.json", identifier, className)), data);
+  }
+
+  private static String getSimpleClassName(URI path) {
+    return Paths.get(path).getFileName().toString().replace(".java", "");
+  }
+}

error-prone-contrib/pom.xml

@@ -179,6 +179,11 @@
             <artifactId>mongodb-driver-core</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-core</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.openrewrite</groupId>
             <artifactId>rewrite-core</artifactId>

error-prone-contrib/src/test/java/tech/picnic/errorprone/refasterrules/ReactorRulesBenchmarks.java

@@ -0,0 +1,111 @@
+package tech.picnic.errorprone.refasterrules;
+
+import com.google.errorprone.refaster.annotation.AfterTemplate;
+import com.google.errorprone.refaster.annotation.BeforeTemplate;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OperationsPerInvocation;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.profile.GCProfiler;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import reactor.core.publisher.Mono;
+import tech.picnic.errorprone.refaster.annotation.Benchmarked;
+import tech.picnic.errorprone.refasterrules.ReactorRules.MonoFromOptionalSwitchIfEmpty;
+
+// XXX: Fix warmup and measurements etc.
+@SuppressWarnings("FieldCanBeFinal") // XXX: Triggers CompilesWithFix!!!
+@State(Scope.Benchmark)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Fork(
+    jvmArgs = {"-Xms256M", "-Xmx256M"},
+    value = 1)
+@Warmup(iterations = 1)
+@Measurement(iterations = 1)
+public class ReactorRulesBenchmarks {
+  public static void main(String[] args) throws RunnerException {
+    String testRegex = Pattern.quote(ReactorRulesBenchmarks.class.getCanonicalName());
+    new Runner(new OptionsBuilder().include(testRegex).addProfiler(GCProfiler.class).build()).run();
+  }
+
+  // XXX: What a benchmarked rule could look like.
+  @Benchmarked
+  static final class MonoFromOptionalSwitchIfEmpty<T> {
+    // XXX: These parameters could perhaps be inferred in the common case.
+    @Benchmarked.Param private final Optional<String> emptyOptional = Optional.empty();
+    @Benchmarked.Param private final Optional<String> nonEmptyOptional = Optional.of("foo");
+    @Benchmarked.Param private final Mono<String> emptyMono = Mono.<String>empty().hide();
+    @Benchmarked.Param private final Mono<String> nonEmptyMono = Mono.just("bar").hide();
+
+    @BeforeTemplate
+    Mono<T> before(Optional<T> optional, Mono<T> mono) {
+      return optional.map(Mono::just).orElse(mono);
+    }
+
+    @AfterTemplate
+    Mono<T> after(Optional<T> optional, Mono<T> mono) {
+      return Mono.justOrEmpty(optional).switchIfEmpty(mono);
+    }
+  }
+
+  // XXX: Variations like this would be generated.
+  public static class MonoFromOptionalSwitchIfEmptyBenchmark extends ReactorRulesBenchmarks {
+    private final MonoFromOptionalSwitchIfEmpty<String> rule = new MonoFromOptionalSwitchIfEmpty();
+    //    private final Optional<String> optional = Optional.of("foo");
+    //    private final Mono<String> mono = Mono.just("bar").hide();
+    private final Optional<String> optional = Optional.empty();
+    private final Mono<String> mono = Mono.<String>empty().hide();
+    private final Mono<String> before = rule.before(optional, mono);
+    private final Mono<String> after = rule.after(optional, mono);
+
+    @Benchmark
+    public Mono<String> before() {
+      return rule.before(optional, mono);
+    }
+
+    @Benchmark
+    public String beforeSubscribe1() {
+      return before.block();
+    }
+
+    // XXX: In the common case the `x100` variant this doesn't add much. Leave out, at least by
+    // default.
+    @Benchmark
+    @OperationsPerInvocation(100)
+    public void beforeSubscribe100(Blackhole bh) {
+      for (int i = 0; i < 100; i++) {
+        bh.consume(before.block());
+      }
+    }
+
+    @Benchmark
+    public Mono<String> after() {
+      return rule.after(optional, mono);
+    }
+
+    @Benchmark
+    public String afterSubscribe() {
+      return after.block();
+    }
+
+    @Benchmark
+    @OperationsPerInvocation(100)
+    public void afterSubscribe100(Blackhole bh) {
+      for (int i = 0; i < 100; i++) {
+        bh.consume(after.block());
+      }
+    }
+  }
+}

pom.xml

@@ -212,6 +212,7 @@
         <version.error-prone-slf4j>0.1.24</version.error-prone-slf4j>
         <version.guava-beta-checker>1.0</version.guava-beta-checker>
         <version.jdk>17</version.jdk>
+        <version.jmh>1.37</version.jmh>
         <version.maven>3.9.5</version.maven>
         <version.mockito>5.11.0</version.mockito>
         <version.nopen-checker>1.0.1</version.nopen-checker>
@@ -468,6 +469,11 @@
                 <artifactId>mongodb-driver-core</artifactId>
                 <version>5.1.0</version>
             </dependency>
+            <dependency>
+                <groupId>org.openjdk.jmh</groupId>
+                <artifactId>jmh-core</artifactId>
+                <version>${version.jmh}</version>
+            </dependency>
             <dependency>
                 <groupId>org.openrewrite</groupId>
                 <artifactId>rewrite-templating</artifactId>
@@ -932,6 +938,11 @@
                                 <artifactId>auto-service</artifactId>
                                 <version>${version.auto-service}</version>
                             </path>
+                            <path>
+                                <groupId>org.openjdk.jmh</groupId>
+                                <artifactId>jmh-generator-annprocess</artifactId>
+                                <version>${version.jmh}</version>
+                            </path>
                             <path>
                                 <groupId>org.openrewrite</groupId>
                                 <artifactId>rewrite-templating</artifactId>
@@ -2054,6 +2065,67 @@
                 </pluginManagement>
             </build>
         </profile>
+        <profile>
+            <!-- Enables execution of a JMH benchmark. Given a benchmark class
+            `tech.picnic.MyBenchmark`, the following command (executed against
+            the (sub)module in which the benchmark resides) will compile and
+            execute said benchmark:
+
+              mvn process-test-classes -Dverification.skip \
+                -Djmh.run-benchmark=tech.picnic.MyBenchmark
+            -->
+            <id>run-jmh-benchmark</id>
+            <activation>
+                <property>
+                    <name>jmh.run-benchmark</name>
+                </property>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-dependency-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>build-jmh-runtime-classpath</id>
+                                <goals>
+                                    <goal>build-classpath</goal>
+                                </goals>
+                                <configuration>
+                                    <outputProperty>testClasspath</outputProperty>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>run-jmh-benchmark</id>
+                                <goals>
+                                    <goal>java</goal>
+                                </goals>
+                                <phase>process-test-classes</phase>
+                                <configuration>
+                                    <classpathScope>test</classpathScope>
+                                    <mainClass>${jmh.run-benchmark}</mainClass>
+                                    <systemProperties>
+                                        <!-- The runtime classpath is defined
+                                        in this way so that any JVMs forked by
+                                        JMH will have the desired classpath. -->
+                                        <systemProperty>
+                                            <key>java.class.path</key>
+                                            <value>${project.build.testOutputDirectory}${path.separator}${project.build.outputDirectory}${path.separator}${testClasspath}</value>
+                                        </systemProperty>
+                                    </systemProperties>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
         <profile>
             <!-- This profile is auto-activated when performing a release. -->
             <id>release</id>

refaster-support/src/main/java/tech/picnic/errorprone/refaster/annotation/Benchmarked.java

@@ -0,0 +1,27 @@
+package tech.picnic.errorprone.refaster.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Indicates that benchmark(s) should be generated for the annotated Refaster rule or group of
+ * Refaster rules.
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.SOURCE)
+public @interface Benchmarked {
+  // XXX: Make configurable. E.g.
+  // - While not supported, don't complain about `Refaster.anyOf` or other `Refaster` utility method
+  //   usages.
+  // - Specify warmup and measurement iterations.
+  // - Specify output time unit.
+  // - Value generation hints.f
+  // Once configuration is supported, annotations on nested classes should override the configuration specified by outer classes.
+
+  // XXX: Explain use. Allow restriction by name?
+  public @interface Param {
+
+  }
+}