Implementation of MP FT 2.1.1 using FT SE (#2348)

* Replacing FailSafe and Hystrix by our own implementation of FT primitives. Some minor changes to our first version of these primitive operations was necessary to be fully compatible with MP and pass all the TCKs.

Signed-off-by: Santiago Pericasgeertsen <santiago.pericasgeertsen@oracle.com>

common/reactive/src/main/java/io/helidon/common/reactive/MultiFromCompletionStage.java

@@ -42,7 +42,7 @@ final class MultiFromCompletionStage<T> implements Multi<T> {
 
     static <T> void subscribe(Flow.Subscriber<? super T> subscriber, CompletionStage<T> source, boolean nullMeansEmpty) {
         AtomicBiConsumer<T> watcher = new AtomicBiConsumer<>();
-        CompletionStageSubscription<T> css = new CompletionStageSubscription<>(subscriber, nullMeansEmpty, watcher);
+        CompletionStageSubscription<T> css = new CompletionStageSubscription<>(subscriber, nullMeansEmpty, watcher, source);
         watcher.lazySet(css);
 
         subscriber.onSubscribe(css);
@@ -55,10 +55,13 @@ final class MultiFromCompletionStage<T> implements Multi<T> {
 
         private final AtomicBiConsumer<T> watcher;
 
-        CompletionStageSubscription(Flow.Subscriber<? super T> downstream, boolean nullMeansEmpty, AtomicBiConsumer<T> watcher) {
+        private CompletionStage<T> source;
+        CompletionStageSubscription(Flow.Subscriber<? super T> downstream, boolean nullMeansEmpty,
+                                    AtomicBiConsumer<T> watcher, CompletionStage<T> source) {
             super(downstream);
             this.nullMeansEmpty = nullMeansEmpty;
             this.watcher = watcher;
+            this.source = source;
         }
 
         @Override
@@ -77,6 +80,7 @@ final class MultiFromCompletionStage<T> implements Multi<T> {
         @Override
         public void cancel() {
             super.cancel();
+            source.toCompletableFuture().cancel(true);
             watcher.getAndSet(null);
         }
     }

common/reactive/src/main/java/io/helidon/common/reactive/Single.java

@@ -648,8 +648,21 @@ public interface Single<T> extends Subscribable<T>, CompletionStage<T>, Awaitabl
      * @return CompletionStage
      */
     default CompletionStage<T> toStage() {
+        return toStage(false);
+    }
+
+    /**
+     * Exposes this {@link Single} instance as a {@link CompletionStage}.
+     * Note that if this {@link Single} completes without a value and {@code completeWithoutValue}
+     * is set to {@code false}, the resulting {@link CompletionStage} will be completed
+     * exceptionally with an {@link IllegalStateException}
+     *
+     * @param completeWithoutValue Allow completion without a value.
+     * @return CompletionStage
+     */
+    default CompletionStage<T> toStage(boolean completeWithoutValue) {
         try {
-            SingleToFuture<T> subscriber = new SingleToFuture<>(this, false);
+            SingleToFuture<T> subscriber = new SingleToFuture<>(this, completeWithoutValue);
             this.subscribe(subscriber);
             return subscriber;
         } catch (Throwable ex) {

dependencies/pom.xml

@@ -85,7 +85,7 @@
         <version.lib.microprofile-jwt>1.1.1</version.lib.microprofile-jwt>
         <version.lib.microprofile-metrics-api>2.3.2</version.lib.microprofile-metrics-api>
         <version.lib.microprofile-openapi-api>1.1.2</version.lib.microprofile-openapi-api>
-        <version.lib.microprofile-fault-tolerance-api>2.0.2</version.lib.microprofile-fault-tolerance-api>
+        <version.lib.microprofile-fault-tolerance-api>2.1.1</version.lib.microprofile-fault-tolerance-api>
         <version.lib.microprofile-tracing>1.3.3</version.lib.microprofile-tracing>
         <version.lib.microprofile-rest-client>1.3.3</version.lib.microprofile-rest-client>
         <version.lib.microprofile-reactive-messaging-api>1.0</version.lib.microprofile-reactive-messaging-api>

examples/webserver/fault-tolerance/src/main/java/io/helidon/webserver/examples/faulttolerance/FtService.java

@@ -51,8 +51,8 @@ public class FtService implements Service {
                 .name("helidon-example-bulkhead")
                 .build();
         this.breaker = CircuitBreaker.builder()
-                .volume(10)
-                .errorRatio(20)
+                .volume(4)
+                .errorRatio(40)
                 .successThreshold(1)
                 .delay(Duration.ofSeconds(5))
                 .build();

fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java

@@ -18,6 +18,7 @@ package io.helidon.faulttolerance;
 
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
 import java.util.function.Supplier;
 
 import io.helidon.common.LazyValue;
@@ -35,14 +36,16 @@ class AsyncImpl implements Async {
         CompletableFuture<T> future = new CompletableFuture<>();
         AsyncTask<T> task = new AsyncTask<>(supplier, future);
 
+        Future<?> taskFuture;
         try {
-            executor.get().submit(task);
+            taskFuture = executor.get().submit(task);
         } catch (Throwable e) {
             // rejected execution and other executor related issues
             return Single.error(e);
         }
 
-        return Single.create(future);
+        Single<T> single = Single.create(future, true);
+        return single.onCancel(() -> taskFuture.cancel(false));     // cancel task
     }
 
     private static class AsyncTask<T> implements Runnable {

fault-tolerance/src/main/java/io/helidon/faulttolerance/AtomicCycle.java

@@ -26,6 +26,14 @@ final class AtomicCycle {
         this.maxIndex = maxIndex + 1;
     }
 
+    int get() {
+        return atomicInteger.get();
+    }
+
+    void set(int n) {
+        atomicInteger.set(n);
+    }
+
     int incrementAndGet() {
         return atomicInteger.accumulateAndGet(maxIndex, (current, max) -> (current + 1) % max);
     }

fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java

@@ -125,4 +125,41 @@ public interface Bulkhead extends FtHandler {
         }
     }
 
+    interface Stats {
+
+        /**
+         * Number of concurrent executions at this time.
+         *
+         * @return concurrent executions.
+         */
+        long concurrentExecutions();
+
+        /**
+         * Number of calls accepted on the bulkhead.
+         *
+         * @return calls accepted.
+         */
+        long callsAccepted();
+
+        /**
+         * Number of calls rejected on the bulkhead.
+         *
+         * @return calls rejected.
+         */
+        long callsRejected();
+
+        /**
+         * Size of waiting queue at this time.
+         *
+         * @return size of waiting queue.
+         */
+        long waitingQueueSize();
+    }
+
+    /**
+     * Provides access to internal stats for this bulkhead.
+     *
+     * @return internal stats.
+     */
+    Stats stats();
 }

fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java

@@ -23,6 +23,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Flow;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
 import java.util.logging.Logger;
 
@@ -38,6 +39,10 @@ class BulkheadImpl implements Bulkhead {
     private final Semaphore inProgress;
     private final String name;
 
+    private final AtomicLong concurrentExecutions = new AtomicLong(0L);
+    private final AtomicLong callsAccepted = new AtomicLong(0L);
+    private final AtomicLong callsRejected = new AtomicLong(0L);
+
     BulkheadImpl(Bulkhead.Builder builder) {
         this.executor = builder.executor();
         this.inProgress = new Semaphore(builder.limit(), true);
@@ -60,10 +65,36 @@ class BulkheadImpl implements Bulkhead {
         return invokeTask(DelayedTask.createMulti(supplier));
     }
 
+    @Override
+    public Stats stats() {
+        return new Stats() {
+            @Override
+            public long concurrentExecutions() {
+                return concurrentExecutions.get();
+            }
+
+            @Override
+            public long callsAccepted() {
+                return callsAccepted.get();
+            }
+
+            @Override
+            public long callsRejected() {
+                return callsRejected.get();
+            }
+
+            @Override
+            public long waitingQueueSize() {
+                return queue.size();
+            }
+        };
+    }
+
     // this method must be called while NOT holding a permit
     private <R> R invokeTask(DelayedTask<R> task) {
         if (inProgress.tryAcquire()) {
             LOGGER.finest(() -> name + " invoke immediate: " + task);
+
             // free permit, we can invoke
             execute(task);
             return task.result();
@@ -71,9 +102,15 @@ class BulkheadImpl implements Bulkhead {
             // no free permit, let's try to enqueue
             if (queue.offer(task)) {
                 LOGGER.finest(() -> name + " enqueue: " + task);
-                return task.result();
+                R result = task.result();
+                if (result instanceof Single<?>) {
+                    Single<Object> single = (Single<Object>) result;
+                    return (R) single.onCancel(() -> queue.remove(task));
+                }
+                return result;
             } else {
                 LOGGER.finest(() -> name + " reject: " + task);
+                callsRejected.incrementAndGet();
                 return task.error(new BulkheadException("Bulkhead queue \"" + name + "\" is full"));
             }
         }
@@ -81,8 +118,12 @@ class BulkheadImpl implements Bulkhead {
 
     // this method must be called while holding a permit
     private void execute(DelayedTask<?> task) {
+        callsAccepted.incrementAndGet();
+        concurrentExecutions.incrementAndGet();
+
         task.execute()
                 .handle((it, throwable) -> {
+                    concurrentExecutions.decrementAndGet();
                     // we do not care about execution, but let's record it in debug
                     LOGGER.finest(() -> name + " finished execution: " + task
                             + " (" + (throwable == null ? "success" : "failure") + ")");

fault-tolerance/src/main/java/io/helidon/faulttolerance/CircuitBreakerImpl.java

@@ -77,21 +77,18 @@ class CircuitBreakerImpl implements CircuitBreaker {
         if (state.get() == State.CLOSED) {
             // run it!
             CompletionStage<Void> completion = task.execute();
-
             completion.handle((it, throwable) -> {
                 Throwable exception = FaultTolerance.cause(throwable);
                 if (exception == null || errorChecker.shouldSkip(exception)) {
-                    // success
                     results.update(SUCCESS);
                 } else {
                     results.update(FAILURE);
-                    if (results.shouldOpen() && state.compareAndSet(State.CLOSED, State.OPEN)) {
-                        results.reset();
-                        // if we successfully switch to open, we need to schedule switch to half-open
-                        scheduleHalf();
-                    }
                 }
-
+                if (results.shouldOpen() && state.compareAndSet(State.CLOSED, State.OPEN)) {
+                    results.reset();
+                    // if we successfully switch to open, we need to schedule switch to half-open
+                    scheduleHalf();
+                }
                 return it;
             });
             return task.result();
@@ -111,18 +108,15 @@ class CircuitBreakerImpl implements CircuitBreaker {
                             // transition to closed
                             successCounter.set(0);
                             state.compareAndSet(State.HALF_OPEN, State.CLOSED);
-                            halfOpenInProgress.set(false);
                         }
-                        halfOpenInProgress.set(false);
                     } else {
                         // failure
                         successCounter.set(0);
                         state.set(State.OPEN);
-                        halfOpenInProgress.set(false);
                         // if we successfully switch to open, we need to schedule switch to half-open
                         scheduleHalf();
                     }
-
+                    halfOpenInProgress.set(false);
                     return it;
                 });
                 return task.result();

fault-tolerance/src/main/java/io/helidon/faulttolerance/DelayedTask.java

@@ -133,7 +133,7 @@ interface DelayedTask<T> {
 
             @Override
             public Single<T> result() {
-                return Single.create(resultFuture.get());
+                return Single.create(resultFuture.get(), true);
             }
 
             @Override

fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java

@@ -22,25 +22,24 @@ import java.util.Set;
 interface ErrorChecker {
     boolean shouldSkip(Throwable throwable);
 
+    /**
+     * Returns ErrorChecker that skips if throwable is in skipOnSet or if applyOnSet
+     * is not empty and throwable is not in it. Note that if applyOnSet is empty, then
+     * it is equivalent to it containing {@code Throwable.class}. Sets are copied
+     * because they are mutable.
+     *
+     * @param skipOnSet set of throwables to skip logic on.
+     * @param applyOnSet set of throwables to apply logic on.
+     * @return An error checker.
+     */
     static ErrorChecker create(Set<Class<? extends Throwable>> skipOnSet, Set<Class<? extends Throwable>> applyOnSet) {
         Set<Class<? extends Throwable>> skipOn = Set.copyOf(skipOnSet);
         Set<Class<? extends Throwable>> applyOn = Set.copyOf(applyOnSet);
+        return throwable -> containsThrowable(skipOn, throwable)
+                || !applyOn.isEmpty() && !containsThrowable(applyOn, throwable);
+    }
 
-        if (skipOn.isEmpty()) {
-            if (applyOn.isEmpty()) {
-                return throwable -> false;
-            } else {
-                return throwable -> !applyOn.contains(throwable.getClass());
-            }
-        } else {
-            if (applyOn.isEmpty()) {
-                return throwable -> skipOn.contains(throwable.getClass());
-            } else {
-                throw new IllegalArgumentException("You have defined both skip and apply set of exception classes. "
-                                                           + "This cannot be correctly handled; skipOn: " + skipOn
-                                                           + " applyOn: " + applyOn);
-            }
-
-        }
+    private static boolean containsThrowable(Set<Class<? extends Throwable>> set, Throwable throwable) {
+        return set.stream().anyMatch(t -> t.isAssignableFrom(throwable.getClass()));
     }
 }

fault-tolerance/src/main/java/io/helidon/faulttolerance/FallbackImpl.java

@@ -76,6 +76,6 @@ class FallbackImpl<T> implements Fallback<T> {
                     return null;
                 });
 
-        return Single.create(future);
+        return Single.create(future, true);
     }
 }

fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java

@@ -121,6 +121,16 @@ public final class FaultTolerance {
         return new Builder();
     }
 
+    /**
+     * A typed builder to configure a customized sequence of fault tolerance handlers.
+     *
+     * @param <T> type of result
+     * @return a new builder
+     */
+    public static <T> TypedBuilder<T> typedBuilder() {
+        return new TypedBuilder<>();
+    }
+
     static Config config() {
         return CONFIG.get();
     }
@@ -266,7 +276,17 @@ public final class FaultTolerance {
                     next = () -> validFt.invoke(finalNext);
                 }
 
-                return Single.create(next.get());
+                return Single.create(next.get(), true);
+            }
+
+            @Override
+            public String toString() {
+                StringBuilder sb = new StringBuilder();
+                for (int i = validFts.size() - 1; i >= 0; i--) {
+                    sb.append(validFts.get(i).toString());
+                    sb.append("\n");
+                }
+                return sb.toString();
             }
         }
 
@@ -286,6 +306,11 @@ public final class FaultTolerance {
             public Multi<T> invokeMulti(Supplier<? extends Flow.Publisher<T>> supplier) {
                 return handler.invokeMulti(supplier);
             }
+
+            @Override
+            public String toString() {
+                return handler.getClass().getSimpleName();
+            }
         }
     }
 
@@ -350,7 +375,7 @@ public final class FaultTolerance {
                     next = () -> validFt.invoke(finalNext);
                 }
 
-                return Single.create(next.get());
+                return Single.create(next.get(), true);
             }
         }
     }

fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java

@@ -29,6 +29,7 @@ final class ResultWindow {
     private final AtomicInteger currentSum = new AtomicInteger();
     private final AtomicCycle index;
     private final AtomicInteger[] results;
+    private final AtomicInteger totalResults = new AtomicInteger();
     private final int thresholdSum;
 
     ResultWindow(int size, int ratio) {
@@ -44,17 +45,18 @@ final class ResultWindow {
     }
 
     void update(Result resultEnum) {
+        // update total number of results
+        totalResults.incrementAndGet();
+
         // success is zero, failure is 1
         int result = resultEnum.ordinal();
 
         AtomicInteger mine = results[index.incrementAndGet()];
         int origValue = mine.getAndSet(result);
-
         if (origValue == result) {
             // no change
             return;
         }
-
         if (origValue == 1) {
             currentSum.decrementAndGet();
         } else {
@@ -62,15 +64,22 @@ final class ResultWindow {
         }
     }
 
+    /**
+     * Open if we have seen enough results and we are at or over the threshold.
+     *
+     * @return outcome of test.
+     */
     boolean shouldOpen() {
-        return currentSum.get() >= thresholdSum;
+        return totalResults.get() >= results.length && currentSum.get() >= thresholdSum;
     }
 
     void reset() {
-        // "soft" reset - send in success equal to window size
         for (int i = 0; i < results.length; i++) {
-            update(Result.SUCCESS);
+            results[i].set(Result.SUCCESS.ordinal());
         }
+        currentSum.set(0);
+        index.set(results.length - 1);
+        totalResults.set(0);
     }
 
     // order is significant, do not change

fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java

@@ -58,7 +58,6 @@ public interface Retry extends FtHandler {
                 .jitter(Duration.ofMillis(50))
                 .build();
 
-
         private Duration overallTimeout = Duration.ofSeconds(1);
         private LazyValue<? extends ScheduledExecutorService> scheduledExecutor = FaultTolerance.scheduledExecutor();
 
@@ -422,4 +421,12 @@ public interface Retry extends FtHandler {
             }
         }
     }
+
+    /**
+     * Number of times a method called has been retried. This is a monotonically
+     * increasing counter over the lifetime of the handler.
+     *
+     * @return number ot times a method is retried.
+     */
+    long retryCounter();
 }

fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java

@@ -37,6 +37,7 @@ class RetryImpl implements Retry {
     private final ErrorChecker errorChecker;
     private final long maxTimeNanos;
     private final Retry.RetryPolicy retryPolicy;
+    private final AtomicLong retryCounter = new AtomicLong(0L);
 
     RetryImpl(Retry.Builder builder) {
         this.scheduledExecutor = builder.scheduledExecutor();
@@ -75,6 +76,10 @@ class RetryImpl implements Retry {
                                                              + TimeUnit.NANOSECONDS.toMillis(maxTimeNanos) + " ms."));
         }
 
+        if (currentCallIndex > 0) {
+            retryCounter.getAndIncrement();
+        }
+
         DelayedTask<Single<T>> task = DelayedTask.createSingle(context.supplier);
         if (delay == 0) {
             task.execute();
@@ -94,7 +99,6 @@ class RetryImpl implements Retry {
     }
 
     private <T> Multi<T> retryMulti(RetryContext<? extends Flow.Publisher<T>> context) {
-
         long delay = 0;
         int currentCallIndex = context.count.getAndIncrement();
         if (currentCallIndex != 0) {
@@ -114,6 +118,10 @@ class RetryImpl implements Retry {
                                                              + TimeUnit.NANOSECONDS.toMillis(maxTimeNanos) + " ms."));
         }
 
+        if (currentCallIndex > 0) {
+            retryCounter.getAndIncrement();
+        }
+
         DelayedTask<Multi<T>> task = DelayedTask.createMulti(context.supplier);
         if (delay == 0) {
             task.execute();
@@ -132,6 +140,11 @@ class RetryImpl implements Retry {
                 });
     }
 
+    @Override
+    public long retryCounter() {
+        return retryCounter.get();
+    }
+
     private static class RetryContext<U> {
         // retry runtime
         private final long startedMillis = System.currentTimeMillis();

fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java

@@ -52,6 +52,7 @@ public interface Timeout extends FtHandler {
     class Builder implements io.helidon.common.Builder<Timeout> {
         private Duration timeout = Duration.ofSeconds(10);
         private LazyValue<? extends ScheduledExecutorService> executor = FaultTolerance.scheduledExecutor();
+        private boolean currentThread = false;
 
         private Builder() {
         }
@@ -72,6 +73,18 @@ public interface Timeout extends FtHandler {
             return this;
         }
 
+        /**
+         * Flag to indicate that code must be executed in current thread instead
+         * of in an executor's thread. This flag is {@code false} by default.
+         *
+         * @param currentThread setting for this timeout
+         * @return updated builder instance
+         */
+        public Builder currentThread(boolean currentThread) {
+            this.currentThread = currentThread;
+            return this;
+        }
+
         /**
          * Executor service to schedule the timeout.
          *
@@ -90,5 +103,9 @@ public interface Timeout extends FtHandler {
         LazyValue<? extends ScheduledExecutorService> executor() {
             return executor;
         }
+
+        boolean currentThread() {
+            return currentThread;
+        }
     }
 }

fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java

@@ -16,10 +16,14 @@
 
 package io.helidon.faulttolerance;
 
+import java.time.Duration;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.Flow;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
 
 import io.helidon.common.LazyValue;
@@ -27,23 +31,78 @@ import io.helidon.common.reactive.Multi;
 import io.helidon.common.reactive.Single;
 
 class TimeoutImpl implements Timeout {
+    private static final long MONITOR_THREAD_TIMEOUT = 100L;
+
     private final long timeoutMillis;
     private final LazyValue<? extends ScheduledExecutorService> executor;
+    private final boolean currentThread;
 
     TimeoutImpl(Timeout.Builder builder) {
         this.timeoutMillis = builder.timeout().toMillis();
         this.executor = builder.executor();
+        this.currentThread = builder.currentThread();
     }
 
     @Override
     public <T> Multi<T> invokeMulti(Supplier<? extends Flow.Publisher<T>> supplier) {
+        if (currentThread) {
+            throw new UnsupportedOperationException("Unsupported currentThread flag with Multi");
+        }
         return Multi.create(supplier.get())
                 .timeout(timeoutMillis, TimeUnit.MILLISECONDS, executor.get());
     }
 
     @Override
     public <T> Single<T> invoke(Supplier<? extends CompletionStage<T>> supplier) {
-        return Single.create(supplier.get())
-                .timeout(timeoutMillis, TimeUnit.MILLISECONDS, executor.get());
+        if (!currentThread) {
+            return Single.create(supplier.get(), true)
+                    .timeout(timeoutMillis, TimeUnit.MILLISECONDS, executor.get());
+        } else {
+            Thread thisThread = Thread.currentThread();
+            CompletableFuture<Void> monitorStarted = new CompletableFuture<>();
+            AtomicBoolean callReturned = new AtomicBoolean(false);
+
+            // Startup monitor thread that can interrupt current thread after timeout
+            CompletableFuture<T> future = new CompletableFuture<>();
+            Timeout.builder()
+                    .executor(executor.get())       // propagate executor
+                    .currentThread(false)
+                    .timeout(Duration.ofMillis(timeoutMillis))
+                    .build()
+                    .invoke(() -> {
+                        monitorStarted.complete(null);
+                        return Single.never();
+                    })
+                    .exceptionally(it -> {
+                        if (callReturned.compareAndSet(false, true)) {
+                            future.completeExceptionally(new TimeoutException("Method interrupted by timeout"));
+                            thisThread.interrupt();
+                        }
+                        return null;
+                    });
+
+            // Ensure monitor thread has started
+            try {
+                monitorStarted.get(MONITOR_THREAD_TIMEOUT, TimeUnit.MILLISECONDS);
+            } catch (Exception e) {
+                return Single.error(new IllegalStateException("Timeout monitor thread failed to start"));
+            }
+
+            // Run invocation in current thread
+            Single<T> single = Single.create(supplier.get(), true);
+            callReturned.set(true);
+            single.whenComplete((o, t) -> {
+                if (t != null) {
+                    future.completeExceptionally(t);
+                } else {
+                    future.complete(o);
+                }
+            });
+
+            // Clear interrupted flag here -- required for uninterruptible busy loops
+            Thread.interrupted();
+
+            return Single.create(future, true);
+        }
     }
 }

fault-tolerance/src/test/java/io/helidon/faulttolerance/CircuitBreakerTest.java

@@ -47,14 +47,14 @@ class CircuitBreakerTest {
 
         good(breaker);
         good(breaker);
-
         bad(breaker);
-
         good(breaker);
         goodMulti(breaker);
-
-        // should open the breaker
+        good(breaker);
+        good(breaker);
+        good(breaker);
         bad(breaker);
+        bad(breaker);       // should open - window complete
 
         breakerOpen(breaker);
         breakerOpenMulti(breaker);
@@ -77,23 +77,19 @@ class CircuitBreakerTest {
 
         assertThat(breaker.state(), is(CircuitBreaker.State.CLOSED));
 
-        // should open the breaker
-        bad(breaker);
-        bad(breaker);
-
-        assertThat(breaker.state(), is(CircuitBreaker.State.OPEN));
-
-        // need to wait until half open
-        count = 0;
-        while (count++ < 10) {
-            Thread.sleep(50);
-            if (breaker.state() == CircuitBreaker.State.HALF_OPEN) {
-                break;
-            }
-        }
-
         good(breaker);
-        badMulti(breaker);
+        good(breaker);
+        bad(breaker);
+        good(breaker);
+        goodMulti(breaker);
+        good(breaker);
+        good(breaker);
+        good(breaker);
+        bad(breaker);
+        bad(breaker);       // should open - window complete
+
+        breakerOpen(breaker);
+        breakerOpenMulti(breaker);
 
         assertThat(breaker.state(), is(CircuitBreaker.State.OPEN));
     }

fault-tolerance/src/test/java/io/helidon/faulttolerance/ResultWindowTest.java

@@ -22,24 +22,66 @@ import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
 
 class ResultWindowTest {
+
     @Test
-    void test() {
-        ResultWindow window = new ResultWindow(10, 10);
+    void testNotOpenBeforeCompleteWindow() {
+        ResultWindow window = new ResultWindow(5, 20);
+        assertThat("Empty should not open", window.shouldOpen(), is(false));
+        window.update(ResultWindow.Result.FAILURE);
+        window.update(ResultWindow.Result.FAILURE);
+        window.update(ResultWindow.Result.FAILURE);
+        window.update(ResultWindow.Result.FAILURE);
+        assertThat("Should not open before complete window", window.shouldOpen(), is(false));
+    }
+
+    @Test
+    void testOpenAfterCompleteWindow1() {
+        ResultWindow window = new ResultWindow(5, 20);
+        assertThat("Empty should not open", window.shouldOpen(), is(false));
+        window.update(ResultWindow.Result.FAILURE);
+        window.update(ResultWindow.Result.FAILURE);
+        window.update(ResultWindow.Result.SUCCESS);
+        window.update(ResultWindow.Result.SUCCESS);
+        window.update(ResultWindow.Result.SUCCESS);
+        assertThat("Should open after complete window > 20%", window.shouldOpen(), is(true));
+    }
+
+    @Test
+    void testOpenAfterCompleteWindow2() {
+        ResultWindow window = new ResultWindow(5, 20);
+        assertThat("Empty should not open", window.shouldOpen(), is(false));
+        window.update(ResultWindow.Result.SUCCESS);
+        window.update(ResultWindow.Result.FAILURE);
+        window.update(ResultWindow.Result.SUCCESS);
+        window.update(ResultWindow.Result.FAILURE);
+        window.update(ResultWindow.Result.SUCCESS);
+        assertThat("Should open after complete window > 20%", window.shouldOpen(), is(true));
+    }
+
+    @Test
+    void testOpenAfterCompleteWindow3() {
+        ResultWindow window = new ResultWindow(5, 20);
         assertThat("Empty should not open", window.shouldOpen(), is(false));
         window.update(ResultWindow.Result.SUCCESS);
         window.update(ResultWindow.Result.SUCCESS);
         window.update(ResultWindow.Result.SUCCESS);
-        assertThat("Only success should not open", window.shouldOpen(), is(false));
+        window.update(ResultWindow.Result.SUCCESS);
         window.update(ResultWindow.Result.FAILURE);
-        assertThat("Should open on first failure (10% of 10 size)", window.shouldOpen(), is(true));
-        //now cycle through window and replace all with success
-        for (int i = 0; i < 10; i++) {
-             window.update(ResultWindow.Result.SUCCESS);
-        }
-        assertThat("All success should not open", window.shouldOpen(), is(false));
         window.update(ResultWindow.Result.FAILURE);
-        assertThat("Should open on first failure (10% of 10 size)", window.shouldOpen(), is(true));
+        assertThat("Should open after complete window > 20%", window.shouldOpen(), is(true));
+    }
+
+    @Test
+    void testOpenAfterCompleteWindowReset() {
+        ResultWindow window = new ResultWindow(5, 20);
+        assertThat("Empty should not open", window.shouldOpen(), is(false));
+        window.update(ResultWindow.Result.FAILURE);
+        window.update(ResultWindow.Result.FAILURE);
+        window.update(ResultWindow.Result.FAILURE);
+        window.update(ResultWindow.Result.FAILURE);
+        window.update(ResultWindow.Result.FAILURE);
+        assertThat("Should open after complete window > 20%", window.shouldOpen(), is(true));
         window.reset();
-        assertThat("Should not open after reset", window.shouldOpen(), is(false));
+        assertThat("Empty should not open", window.shouldOpen(), is(false));
     }
 }

fault-tolerance/src/test/java/io/helidon/faulttolerance/RetryTest.java

@@ -139,15 +139,6 @@ class RetryTest {
         assertThat("Should have been called twice", req.call.get(), isOneOf(1, 2));
     }
 
-    @Test
-    void testBadConfiguration() {
-        Retry.Builder builder = Retry.builder()
-                .applyOn(RetryException.class)
-                .skipOn(TerminalException.class);
-
-        assertThrows(IllegalArgumentException.class, builder::build);
-    }
-
     @Test
     void testMultiRetriesNoFailure() throws InterruptedException {
         Retry retry = Retry.builder()

microprofile/fault-tolerance/pom.xml

@@ -58,19 +58,15 @@
             <groupId>org.eclipse.microprofile.fault-tolerance</groupId>
             <artifactId>microprofile-fault-tolerance-api</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.helidon.fault-tolerance</groupId>
+            <artifactId>helidon-fault-tolerance</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.eclipse.microprofile.metrics</groupId>
             <artifactId>microprofile-metrics-api</artifactId>
             <scope>provided</scope>
         </dependency>
-        <dependency>
-            <groupId>com.netflix.hystrix</groupId>
-            <artifactId>hystrix-core</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>net.jodah</groupId>
-            <artifactId>failsafe</artifactId>
-        </dependency>
         <dependency>
             <groupId>jakarta.enterprise</groupId>
             <artifactId>jakarta.enterprise.cdi-api</artifactId>

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/BulkheadHelper.java

@@ -1,196 +0,0 @@
-/*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.helidon.microprofile.faulttolerance;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.eclipse.microprofile.faulttolerance.Bulkhead;
-
-/**
- * Helper class to keep track of invocations associated with a bulkhead.
- */
-public class BulkheadHelper {
-
-    /**
-     * A command ID is unique to a target (object) and method. A {@link
-     * FaultToleranceCommand} instance is created for each invocation of that
-     * target/method pair and is assigned the same invocation ID. This class
-     * collects information about all those invocations, including their state:
-     * waiting or running.
-     */
-    static class InvocationData {
-
-        /**
-         * Maximum number of concurrent invocations.
-         */
-        private final int maxRunningInvocations;
-
-        /**
-         * The waiting queue size.
-         */
-        private final int waitingQueueSize;
-
-        /**
-         * All invocations in running state. Must be a subset of {@link #allInvocations}.
-         */
-        private Set<FaultToleranceCommand> runningInvocations = new HashSet<>();
-
-        /**
-         * All invocations associated with a invocation.
-         */
-        private Set<FaultToleranceCommand> allInvocations = new HashSet<>();
-
-        InvocationData(int maxRunningCommands, int waitingQueueSize) {
-            this.maxRunningInvocations = maxRunningCommands;
-            this.waitingQueueSize = waitingQueueSize;
-        }
-
-        synchronized boolean isWaitingQueueFull() {
-            return waitingInvocations() == waitingQueueSize;
-        }
-
-        synchronized boolean isAtMaxRunningInvocations() {
-            return runningInvocations.size() == maxRunningInvocations;
-        }
-
-        synchronized void trackInvocation(FaultToleranceCommand invocation) {
-            allInvocations.add(invocation);
-        }
-
-        synchronized void untrackInvocation(FaultToleranceCommand invocation) {
-            allInvocations.remove(invocation);
-        }
-
-        synchronized int runningInvocations() {
-            return runningInvocations.size();
-        }
-
-        synchronized void markAsRunning(FaultToleranceCommand invocation) {
-            runningInvocations.add(invocation);
-        }
-
-        synchronized void markAsNotRunning(FaultToleranceCommand invocation) {
-            runningInvocations.remove(invocation);
-        }
-
-        synchronized int waitingInvocations() {
-            return allInvocations.size() - runningInvocations.size();
-        }
-    }
-
-    /**
-     * Tracks all invocations associated with a command ID.
-     */
-    private static final Map<String, InvocationData> COMMAND_STATS = new ConcurrentHashMap<>();
-
-    /**
-     * Command key.
-     */
-    private final String commandKey;
-
-    /**
-     * Annotation instance.
-     */
-    private final Bulkhead bulkhead;
-
-    BulkheadHelper(String commandKey, Bulkhead bulkhead) {
-        this.commandKey = commandKey;
-        this.bulkhead = bulkhead;
-    }
-
-    private InvocationData invocationData() {
-        return COMMAND_STATS.computeIfAbsent(
-                commandKey,
-                d -> new InvocationData(bulkhead.value(), bulkhead.waitingTaskQueue()));
-    }
-
-    /**
-     * Track a new invocation instance related to a key.
-     */
-    void trackInvocation(FaultToleranceCommand invocation) {
-        invocationData().trackInvocation(invocation);
-    }
-
-    /**
-     * Stop tracking a invocation instance.
-     */
-    void untrackInvocation(FaultToleranceCommand invocation) {
-        invocationData().untrackInvocation(invocation);
-
-        // Attempt to cleanup state when not in use
-        if (runningInvocations() == 0 && waitingInvocations() == 0) {
-            COMMAND_STATS.remove(commandKey);
-        }
-    }
-
-    /**
-     * Mark a invocation instance as running.
-     */
-    void markAsRunning(FaultToleranceCommand invocation) {
-        invocationData().markAsRunning(invocation);
-    }
-
-    /**
-     * Mark a invocation instance as terminated.
-     */
-    void markAsNotRunning(FaultToleranceCommand invocation) {
-        invocationData().markAsNotRunning(invocation);
-    }
-
-    /**
-     * Get the number of invocations that are running.
-     *
-     * @return Number of invocations running.
-     */
-    int runningInvocations() {
-        return invocationData().runningInvocations();
-    }
-
-    /**
-     * Get the number of invocations that are waiting.
-     *
-     * @return Number of invocations waiting.
-     */
-    int waitingInvocations() {
-        return invocationData().waitingInvocations();
-    }
-
-    /**
-     * Check if the invocation queue is full.
-     *
-     * @return Outcome of test.
-     */
-    boolean isWaitingQueueFull() {
-        return invocationData().isWaitingQueueFull();
-    }
-
-    /**
-     * Check if at maximum number of running invocations.
-     *
-     * @return Outcome of test.
-     */
-    boolean isAtMaxRunningInvocations() {
-        return invocationData().isAtMaxRunningInvocations();
-    }
-
-    boolean isInvocationRunning(FaultToleranceCommand command) {
-        return invocationData().runningInvocations.contains(command);
-    }
-}

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CircuitBreakerAntn.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -58,13 +58,6 @@ public class CircuitBreakerAntn extends MethodAntn implements CircuitBreaker {
         }
     }
 
-    @Override
-    public Class<? extends Throwable>[] failOn() {
-        LookupResult<CircuitBreaker> lookupResult = lookupAnnotation(CircuitBreaker.class);
-        final String override = getParamOverride("failOn", lookupResult.getType());
-        return override != null ? parseThrowableArray(override) : lookupResult.getAnnotation().failOn();
-    }
-
     @Override
     public long delay() {
         LookupResult<CircuitBreaker> lookupResult = lookupAnnotation(CircuitBreaker.class);
@@ -99,4 +92,18 @@ public class CircuitBreakerAntn extends MethodAntn implements CircuitBreaker {
         final String override = getParamOverride("successThreshold", lookupResult.getType());
         return override != null ? Integer.parseInt(override) : lookupResult.getAnnotation().successThreshold();
     }
+
+    @Override
+    public Class<? extends Throwable>[] failOn() {
+        LookupResult<CircuitBreaker> lookupResult = lookupAnnotation(CircuitBreaker.class);
+        final String override = getParamOverride("failOn", lookupResult.getType());
+        return override != null ? parseThrowableArray(override) : lookupResult.getAnnotation().failOn();
+    }
+
+    @Override
+    public Class<? extends Throwable>[] skipOn() {
+        LookupResult<CircuitBreaker> lookupResult = lookupAnnotation(CircuitBreaker.class);
+        final String override = getParamOverride("skipOn", lookupResult.getType());
+        return override != null ? parseThrowableArray(override) : lookupResult.getAnnotation().skipOn();
+    }
 }

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CircuitBreakerHelper.java

@@ -1,305 +0,0 @@
-/*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.helidon.microprofile.faulttolerance;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.logging.Logger;
-
-import com.netflix.config.ConfigurationManager;
-import org.eclipse.microprofile.faulttolerance.CircuitBreaker;
-
-/**
- * A CircuitBreakerHelper keeps track of internal states, success and failure
- * ratios for, etc. for all commands. Similar computations are done internally
- * in Hystrix, but we cannot easily access them.
- */
-public class CircuitBreakerHelper {
-    private static final Logger LOGGER = Logger.getLogger(CircuitBreakerHelper.class.getName());
-
-    private static final String FORCE_OPEN = "hystrix.command.%s.circuitBreaker.forceOpen";
-    private static final String FORCE_CLOSED = "hystrix.command.%s.circuitBreaker.forceClosed";
-
-    /**
-     * Internal state of a circuit breaker. We need to track this to implement
-     * a different HALF_OPEN_MP to CLOSED_MP transition than the default in Hystrix.
-     */
-    enum State {
-        CLOSED_MP(0),
-        HALF_OPEN_MP(1),
-        OPEN_MP(2);
-
-        private int value;
-
-        State(int value) {
-            this.value = value;
-        }
-    }
-
-    /**
-     * Data associated with a command for the purpose of tracking a circuit
-     * breaker state.
-     */
-    static class CommandData {
-
-        private int size;
-
-        private final boolean[] results;
-
-        private State state = State.CLOSED_MP;
-
-        private int successCount;
-
-        private long[] inStateNanos = new long[State.values().length];
-
-        private long lastNanosRead;
-
-        private ReentrantLock lock = new ReentrantLock();
-
-        CommandData(int capacity) {
-            results = new boolean[capacity];
-            size = 0;
-            successCount = 0;
-            lastNanosRead = System.nanoTime();
-        }
-
-        ReentrantLock getLock() {
-            return lock;
-        }
-
-        State getState() {
-            return state;
-        }
-
-        long getCurrentStateNanos() {
-            return System.nanoTime() - lastNanosRead;
-        }
-
-        void setState(State newState) {
-            if (state != newState) {
-                updateStateNanos(state);
-                if (newState == State.HALF_OPEN_MP) {
-                    successCount = 0;
-                }
-                state = newState;
-            }
-        }
-
-        long getInStateNanos(State queryState) {
-            if (state == queryState) {
-                updateStateNanos(state);
-            }
-            return inStateNanos[queryState.value];
-        }
-
-        private void updateStateNanos(State state) {
-            long currentNanos = System.nanoTime();
-            inStateNanos[state.value] += currentNanos - lastNanosRead;
-            lastNanosRead = currentNanos;
-        }
-
-        int getSuccessCount() {
-            return successCount;
-        }
-
-        void incSuccessCount() {
-            successCount++;
-        }
-
-        boolean isAtCapacity() {
-            return size == results.length;
-        }
-
-        void pushResult(boolean result) {
-            if (isAtCapacity()) {
-                shift();
-            }
-            results[size++] = result;
-        }
-
-        double getSuccessRatio() {
-            if (isAtCapacity()) {
-                int success = 0;
-                for (int k = 0; k < size; k++) {
-                    if (results[k]) success++;
-                }
-                return ((double) success) / size;
-            }
-            return -1.0;
-        }
-
-        double getFailureRatio() {
-            double successRatio = getSuccessRatio();
-            return successRatio >= 0.0 ? 1.0 - successRatio : -1.0;
-        }
-
-        private void shift() {
-            if (size > 0) {
-                for (int k = 0; k < size - 1; k++) {
-                    results[k] = results[k + 1];
-                }
-                size--;
-            }
-        }
-    }
-
-    private static final Map<String, CommandData> COMMAND_STATS = new ConcurrentHashMap<>();
-
-    private final FaultToleranceCommand command;
-
-    private final CircuitBreaker circuitBreaker;
-
-    CircuitBreakerHelper(FaultToleranceCommand command, CircuitBreaker circuitBreaker) {
-        this.command = command;
-        this.circuitBreaker = circuitBreaker;
-    }
-
-    private CommandData getCommandData() {
-        return COMMAND_STATS.computeIfAbsent(
-                command.getCommandKey().toString(),
-                d -> new CommandData(circuitBreaker.requestVolumeThreshold()));
-    }
-
-    /**
-     * Reset internal state of command data. Normally, this should be called when
-     * returning to {@link State#CLOSED_MP} state. Since this is the same as the
-     * initial state, we remove it from the map and re-create it later if needed.
-     */
-    void resetCommandData() {
-        ReentrantLock lock = getCommandData().getLock();
-        if (lock.isLocked()) {
-            lock.unlock();
-        }
-        COMMAND_STATS.remove(command.getCommandKey().toString());
-        LOGGER.info("Discarded command data for " + command.getCommandKey());
-    }
-
-    /**
-     * Push a new result into the current window. Discards oldest result
-     * if window is full.
-     *
-     * @param result New result to push.
-     */
-    void pushResult(boolean result) {
-        getCommandData().pushResult(result);
-    }
-
-    /**
-     * Returns nanos since switching to current state.
-     *
-     * @return Nanos in state.
-     */
-    long getCurrentStateNanos() {
-        return getCommandData().getCurrentStateNanos();
-    }
-
-    /**
-     * Computes failure ratio over a complete window.
-     *
-     * @return Failure ratio or -1 if window is not complete.
-     */
-    double getFailureRatio() {
-        return getCommandData().getFailureRatio();
-    }
-
-    /**
-     * Returns state of circuit breaker.
-     *
-     * @return The state.
-     */
-    State getState() {
-        return getCommandData().getState();
-    }
-
-    /**
-     * Changes the state of a circuit breaker.
-     *
-     * @param newState New state.
-     */
-    void setState(State newState) {
-        getCommandData().setState(newState);
-        if (newState == State.OPEN_MP) {
-            openBreaker();
-        } else {
-            closeBreaker();
-        }
-        LOGGER.info("Circuit breaker for " + command.getCommandKey() + " now in state " + getState());
-    }
-
-    /**
-     * Gets success count for breaker.
-     *
-     * @return Success count.
-     */
-    int getSuccessCount() {
-        return getCommandData().getSuccessCount();
-    }
-
-    /**
-     * Increments success counter for breaker.
-     */
-    void incSuccessCount() {
-        getCommandData().incSuccessCount();
-    }
-
-    /**
-     * Prevent concurrent access to underlying command data.
-     */
-    void lock() {
-        getCommandData().getLock().lock();
-    }
-
-    /**
-     * Unlock access to underlying command data.
-     */
-    void unlock() {
-        getCommandData().getLock().unlock();
-    }
-
-    /**
-     * Returns nanos spent on each state.
-     *
-     * @param queryState The state.
-     * @return The time spent in nanos.
-     */
-    long getInStateNanos(State queryState) {
-        return getCommandData().getInStateNanos(queryState);
-    }
-
-    /**
-     * Force Hystrix's circuit breaker into an open state.
-     */
-    private void openBreaker() {
-        if (!command.isCircuitBreakerOpen()) {
-            ConfigurationManager.getConfigInstance().setProperty(
-                    String.format(FORCE_OPEN, command.getCommandKey()), "true");
-        }
-    }
-
-    /**
-     * Force Hystrix's circuit breaker into a closed state.
-     */
-    private void closeBreaker() {
-        if (command.isCircuitBreakerOpen()) {
-            ConfigurationManager.getConfigInstance().setProperty(
-                    String.format(FORCE_OPEN, command.getCommandKey()), "false");
-            ConfigurationManager.getConfigInstance().setProperty(
-                    String.format(FORCE_CLOSED, command.getCommandKey()), "true");
-        }
-    }
-}

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandCompletableFuture.java

@@ -1,348 +0,0 @@
-/*
- * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package io.helidon.microprofile.faulttolerance;
-
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionStage;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.function.BiConsumer;
-import java.util.function.BiFunction;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import java.util.function.Supplier;
-
-/**
- * A wrapper {@link CompletableFuture} which also records the associated {@link FaultToleranceCommand} so
- * {@link CommandScheduler} can retrieve that command. If the delegate returns a result of type
- * {@code Future} then this implementation further unwraps the delegate's value to return the actual
- * value.
- *
- * @param <T> type of result conveyed
- */
-class CommandCompletableFuture<T> extends CompletableFuture<T> {
-
-    static <U> CommandCompletableFuture<U> create(CompletableFuture<U> delegate,
-            Supplier<FaultToleranceCommand> commandSupplier) {
-        return new CommandCompletableFuture<>(delegate, commandSupplier);
-    }
-
-    private final CompletableFuture<T> delegate;
-    private final Supplier<FaultToleranceCommand> commandSupplier;
-
-    private CommandCompletableFuture(CompletableFuture<T> delegate,
-                                     Supplier<FaultToleranceCommand> commandSupplier) {
-        this.delegate = delegate;
-        this.commandSupplier = commandSupplier;
-    }
-
-    @Override
-    public boolean isDone() {
-        return delegate.isDone();
-    }
-
-    @Override
-    public T get() throws InterruptedException, ExecutionException {
-        try {
-            return getResult(-1L, null);
-        } catch (TimeoutException e) {
-            throw new RuntimeException(e); // should never be thrown
-        }
-    }
-
-    @Override
-    public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
-        return getResult(timeout, unit);
-    }
-
-    @SuppressWarnings("unchecked")
-    T getResult(long timeout, TimeUnit unit) throws ExecutionException, InterruptedException, TimeoutException {
-        Object result = timeout < 0 ? delegate.get() : delegate.get(timeout, unit);
-        if (result instanceof CompletionStage<?>) {
-            result = ((CompletionStage<T>) result).toCompletableFuture();
-        }
-        if (result instanceof Future<?>) {
-            final Future<T> future = (Future<T>) result;
-            return timeout < 0 ? future.get() : future.get(timeout, unit);
-        }
-        return (T) result;
-    }
-
-    @Override
-    public T join() {
-        return delegate.join();
-    }
-
-    @Override
-    public T getNow(T valueIfAbsent) {
-        return delegate.getNow(valueIfAbsent);
-    }
-
-    @Override
-    public boolean complete(T value) {
-
-        return delegate.complete(value);
-    }
-
-    @Override
-    public boolean completeExceptionally(Throwable ex) {
-        return delegate.completeExceptionally(ex);
-    }
-
-    @Override
-    public <U> CompletableFuture<U> thenApply(Function<? super T, ? extends U> fn) {
-        return delegate.thenApply(fn);
-    }
-
-    @Override
-    public <U> CompletableFuture<U> thenApplyAsync(Function<? super T, ? extends U> fn) {
-        return delegate.thenApplyAsync(fn);
-    }
-
-    @Override
-    public <U> CompletableFuture<U> thenApplyAsync(Function<? super T, ? extends U> fn, Executor executor) {
-        return delegate.thenApplyAsync(fn, executor);
-    }
-
-    @Override
-    public CompletableFuture<Void> thenAccept(Consumer<? super T> action) {
-        return delegate.thenAccept(action);
-    }
-
-    @Override
-    public CompletableFuture<Void> thenAcceptAsync(Consumer<? super T> action) {
-        return delegate.thenAcceptAsync(action);
-    }
-
-    @Override
-    public CompletableFuture<Void> thenAcceptAsync(Consumer<? super T> action, Executor executor) {
-        return delegate.thenAcceptAsync(action, executor);
-    }
-
-    @Override
-    public CompletableFuture<Void> thenRun(Runnable action) {
-        return delegate.thenRun(action);
-    }
-
-    @Override
-    public CompletableFuture<Void> thenRunAsync(Runnable action) {
-        return delegate.thenRunAsync(action);
-    }
-
-    @Override
-    public CompletableFuture<Void> thenRunAsync(Runnable action, Executor executor) {
-        return delegate.thenRunAsync(action, executor);
-    }
-
-    @Override
-    public <U, V> CompletableFuture<V> thenCombine(CompletionStage<? extends U> other,
-                                                   BiFunction<? super T, ? super U, ? extends V> fn) {
-        return delegate.thenCombine(other, fn);
-    }
-
-    @Override
-    public <U, V> CompletableFuture<V> thenCombineAsync(CompletionStage<? extends U> other,
-                                                        BiFunction<? super T, ? super U, ? extends V> fn) {
-        return delegate.thenCombineAsync(other, fn);
-    }
-
-    @Override
-    public <U, V> CompletableFuture<V> thenCombineAsync(CompletionStage<? extends U> other,
-                                                        BiFunction<? super T, ? super U, ? extends V> fn,
-                                                        Executor executor) {
-        return delegate.thenCombineAsync(other, fn, executor);
-    }
-
-    @Override
-    public <U> CompletableFuture<Void> thenAcceptBoth(CompletionStage<? extends U> other,
-                                                      BiConsumer<? super T, ? super U> action) {
-        return delegate.thenAcceptBoth(other, action);
-    }
-
-    @Override
-    public <U> CompletableFuture<Void> thenAcceptBothAsync(CompletionStage<? extends U> other,
-                                                           BiConsumer<? super T, ? super U> action) {
-        return delegate.thenAcceptBothAsync(other, action);
-    }
-
-    @Override
-    public <U> CompletableFuture<Void> thenAcceptBothAsync(CompletionStage<? extends U> other,
-                                                           BiConsumer<? super T, ? super U> action, Executor executor) {
-        return delegate.thenAcceptBothAsync(other, action, executor);
-    }
-
-    @Override
-    public CompletableFuture<Void> runAfterBoth(CompletionStage<?> other, Runnable action) {
-        return delegate.runAfterBoth(other, action);
-    }
-
-    @Override
-    public CompletableFuture<Void> runAfterBothAsync(CompletionStage<?> other, Runnable action) {
-        return delegate.runAfterBothAsync(other, action);
-    }
-
-    @Override
-    public CompletableFuture<Void> runAfterBothAsync(CompletionStage<?> other, Runnable action, Executor executor) {
-        return delegate.runAfterBothAsync(other, action, executor);
-    }
-
-    @Override
-    public <U> CompletableFuture<U> applyToEither(CompletionStage<? extends T> other, Function<? super T, U> fn) {
-        return delegate.applyToEither(other, fn);
-    }
-
-    @Override
-    public <U> CompletableFuture<U> applyToEitherAsync(CompletionStage<? extends T> other, Function<? super T, U> fn) {
-        return delegate.applyToEitherAsync(other, fn);
-    }
-
-    @Override
-    public <U> CompletableFuture<U> applyToEitherAsync(CompletionStage<? extends T> other, Function<? super T, U> fn,
-                                                       Executor executor) {
-        return delegate.applyToEitherAsync(other, fn, executor);
-    }
-
-    @Override
-    public CompletableFuture<Void> acceptEither(CompletionStage<? extends T> other, Consumer<? super T> action) {
-        return delegate.acceptEither(other, action);
-    }
-
-    @Override
-    public CompletableFuture<Void> acceptEitherAsync(CompletionStage<? extends T> other, Consumer<? super T> action) {
-        return delegate.acceptEitherAsync(other, action);
-    }
-
-    @Override
-    public CompletableFuture<Void> acceptEitherAsync(CompletionStage<? extends T> other, Consumer<? super T> action,
-                                                     Executor executor) {
-        return delegate.acceptEitherAsync(other, action, executor);
-    }
-
-    @Override
-    public CompletableFuture<Void> runAfterEither(CompletionStage<?> other, Runnable action) {
-        return delegate.runAfterEither(other, action);
-    }
-
-    @Override
-    public CompletableFuture<Void> runAfterEitherAsync(CompletionStage<?> other, Runnable action) {
-        return delegate.runAfterEitherAsync(other, action);
-    }
-
-    @Override
-    public CompletableFuture<Void> runAfterEitherAsync(CompletionStage<?> other, Runnable action, Executor executor) {
-        return delegate.runAfterEitherAsync(other, action, executor);
-    }
-
-    @Override
-    public <U> CompletableFuture<U> thenCompose(Function<? super T, ? extends CompletionStage<U>> fn) {
-        return delegate.thenCompose(fn);
-    }
-
-    @Override
-    public <U> CompletableFuture<U> thenComposeAsync(Function<? super T, ? extends CompletionStage<U>> fn) {
-        return delegate.thenComposeAsync(fn);
-    }
-
-    @Override
-    public <U> CompletableFuture<U> thenComposeAsync(Function<? super T, ? extends CompletionStage<U>> fn,
-                                                     Executor executor) {
-        return delegate.thenComposeAsync(fn, executor);
-    }
-
-    @Override
-    public CompletableFuture<T> whenComplete(BiConsumer<? super T, ? super Throwable> action) {
-        return delegate.whenComplete(action);
-    }
-
-    @Override
-    public CompletableFuture<T> whenCompleteAsync(BiConsumer<? super T, ? super Throwable> action) {
-        return delegate.whenCompleteAsync(action);
-    }
-
-    @Override
-    public CompletableFuture<T> whenCompleteAsync(BiConsumer<? super T, ? super Throwable> action, Executor executor) {
-        return delegate.whenCompleteAsync(action, executor);
-    }
-
-    @Override
-    public <U> CompletableFuture<U> handle(BiFunction<? super T, Throwable, ? extends U> fn) {
-        return delegate.handle(fn);
-    }
-
-    @Override
-    public <U> CompletableFuture<U> handleAsync(BiFunction<? super T, Throwable, ? extends U> fn) {
-        return delegate.handleAsync(fn);
-    }
-
-    @Override
-    public <U> CompletableFuture<U> handleAsync(BiFunction<? super T, Throwable, ? extends U> fn, Executor executor) {
-        return delegate.handleAsync(fn, executor);
-    }
-
-    @Override
-    public CompletableFuture<T> toCompletableFuture() {
-        return this;
-    }
-
-    @Override
-    public CompletableFuture<T> exceptionally(Function<Throwable, ? extends T> fn) {
-        return delegate.exceptionally(fn);
-    }
-
-    @Override
-    public boolean cancel(boolean mayInterruptIfRunning) {
-        FaultToleranceCommand command = commandSupplier.get();
-        BulkheadHelper bulkheadHelper = command.getBulkheadHelper();
-        if (bulkheadHelper != null && !bulkheadHelper.isInvocationRunning(command)) {
-            return delegate.cancel(true);      // overridden
-        }
-        return delegate.cancel(mayInterruptIfRunning);
-    }
-
-    @Override
-    public boolean isCancelled() {
-        return delegate.isCancelled();
-    }
-
-    @Override
-    public boolean isCompletedExceptionally() {
-        return delegate.isCompletedExceptionally();
-    }
-
-    @Override
-    public void obtrudeValue(T value) {
-        delegate.obtrudeValue(value);
-    }
-
-    @Override
-    public void obtrudeException(Throwable ex) {
-        delegate.obtrudeException(ex);
-    }
-
-    @Override
-    public int getNumberOfDependents() {
-        return delegate.getNumberOfDependents();
-    }
-
-    @Override
-    public String toString() {
-        return String.format("%s@%h around %s", getClass().getName(), this, delegate.toString());
-    }
-}

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,13 +22,10 @@ import java.lang.reflect.Method;
 import javax.enterprise.inject.spi.CDI;
 import javax.interceptor.InvocationContext;
 
-import net.jodah.failsafe.event.ExecutionAttemptedEvent;
 import org.eclipse.microprofile.faulttolerance.ExecutionContext;
 import org.eclipse.microprofile.faulttolerance.Fallback;
 import org.eclipse.microprofile.faulttolerance.FallbackHandler;
 
-import static io.helidon.microprofile.faulttolerance.ExceptionUtil.toException;
-
 /**
  * Class CommandFallback.
  */
@@ -47,11 +44,11 @@ class CommandFallback {
      *
      * @param context Invocation context.
      * @param introspector Method introspector.
-     * @param event ExecutionAttemptedEvent representing the failure causing the fallback invocation
+     * @param throwable Throwable that caused execution of fallback
      */
-    CommandFallback(InvocationContext context, MethodIntrospector introspector, ExecutionAttemptedEvent<?> event) {
+    CommandFallback(InvocationContext context, MethodIntrospector introspector, Throwable throwable) {
         this.context = context;
-        this.throwable = event.getLastFailure();
+        this.throwable = throwable;
 
         // Establish fallback strategy
         final Fallback fallback = introspector.getFallback();
@@ -107,30 +104,24 @@ class CommandFallback {
                 result = fallbackMethod.invoke(context.getTarget(), context.getParameters());
             }
         } catch (Throwable t) {
-            updateMetrics(t);
+            updateMetrics();
 
             // If InvocationTargetException, then unwrap underlying cause
             if (t instanceof InvocationTargetException) {
                 t = t.getCause();
             }
-            throw toException(t);
+            throw t instanceof Exception ? (Exception) t : new RuntimeException(t);
         }
 
-        updateMetrics(null);
+        updateMetrics();
         return result;
     }
 
     /**
-     * Updates fallback metrics and adjust failed invocations based on outcome of fallback.
+     * Updates fallback metrics.
      */
-    private void updateMetrics(Throwable throwable) {
-        final Method method = context.getMethod();
+    private void updateMetrics() {
+        Method method = context.getMethod();
         FaultToleranceMetrics.getCounter(method, FaultToleranceMetrics.FALLBACK_CALLS_TOTAL).inc();
-
-        // If fallback was successful, it is not a failed invocation
-        if (throwable == null) {
-            // Since metrics 2.0, countes should only be incrementing, so we cheat here
-            FaultToleranceMetrics.getCounter(method, FaultToleranceMetrics.INVOCATIONS_FAILED_TOTAL).inc(-1L);
-        }
     }
 }

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,7 +24,7 @@ import javax.interceptor.Interceptor;
 import javax.interceptor.InvocationContext;
 
 /**
- * Class CommandInterceptor.
+ * Intercepts calls to FT methods and implements annotation semantics.
  */
 @Interceptor
 @CommandBinding
@@ -48,10 +48,10 @@ public class CommandInterceptor {
                         + "::" + context.getMethod().getName() + "'");
 
             // Create method introspector and executer retrier
-            final MethodIntrospector introspector = new MethodIntrospector(
-                    context.getTarget().getClass(), context.getMethod());
-            final CommandRetrier retrier = new CommandRetrier(context, introspector);
-            return retrier.execute();
+            MethodIntrospector introspector = new MethodIntrospector(context.getTarget().getClass(),
+                    context.getMethod());
+            MethodInvoker runner = new MethodInvoker(context, introspector);
+            return runner.get();
         } catch (Throwable t) {
             LOGGER.fine("Throwable caught by interceptor '" + t.getMessage() + "'");
             throw t;

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRetrier.java

@@ -1,467 +0,0 @@
-/*
- * Copyright (c) 2018, 2020 Oracle and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.helidon.microprofile.faulttolerance;
-
-import java.lang.reflect.Method;
-import java.time.Duration;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionStage;
-import java.util.concurrent.Future;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.function.Function;
-import java.util.logging.Logger;
-
-import javax.interceptor.InvocationContext;
-
-import com.netflix.config.ConfigurationManager;
-import com.netflix.hystrix.exception.HystrixRuntimeException;
-import net.jodah.failsafe.Failsafe;
-import net.jodah.failsafe.FailsafeException;
-import net.jodah.failsafe.FailsafeExecutor;
-import net.jodah.failsafe.Fallback;
-import net.jodah.failsafe.Policy;
-import net.jodah.failsafe.RetryPolicy;
-import net.jodah.failsafe.event.ExecutionAttemptedEvent;
-import net.jodah.failsafe.function.CheckedFunction;
-import net.jodah.failsafe.util.concurrent.Scheduler;
-import org.apache.commons.configuration.AbstractConfiguration;
-import org.eclipse.microprofile.config.Config;
-import org.eclipse.microprofile.config.ConfigProvider;
-import org.eclipse.microprofile.faulttolerance.Retry;
-import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException;
-import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException;
-
-import static io.helidon.microprofile.faulttolerance.ExceptionUtil.toException;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceExtension.isFaultToleranceMetricsEnabled;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_CALLS_ACCEPTED_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_CALLS_REJECTED_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_EXECUTION_DURATION;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.INVOCATIONS_FAILED_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.INVOCATIONS_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_FAILED_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_SUCCEEDED_NOT_RETRIED_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_RETRIES_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.TIMEOUT_CALLS_NOT_TIMED_OUT_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.TIMEOUT_CALLS_TIMED_OUT_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.TIMEOUT_EXECUTION_DURATION;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getCounter;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getHistogram;
-
-/**
- * Class CommandRetrier.
- */
-public class CommandRetrier {
-    private static final Logger LOGGER = Logger.getLogger(CommandRetrier.class.getName());
-
-    private static final long   DEFAULT_DELAY_CORRECTION = 250L;
-    private static final String FT_DELAY_CORRECTION = "fault-tolerance.delayCorrection";
-    private static final int    DEFAULT_COMMAND_THREAD_POOL_SIZE = 8;
-    private static final String FT_COMMAND_THREAD_POOL_SIZE = "fault-tolerance.commandThreadPoolSize";
-    private static final long   DEFAULT_THREAD_WAITING_PERIOD = 2000L;
-    private static final String FT_THREAD_WAITING_PERIOD = "fault-tolerance.threadWaitingPeriod";
-    private static final long   DEFAULT_BULKHEAD_TASK_QUEUEING_PERIOD = 2000L;
-    private static final String FT_BULKHEAD_TASK_QUEUEING_PERIOD = "fault-tolerance.bulkheadTaskQueueingPeriod";
-
-    private final InvocationContext context;
-
-    private final RetryPolicy<Object> retryPolicy;
-
-    private final boolean isAsynchronous;
-
-    private final MethodIntrospector introspector;
-
-    private final Method method;
-
-    private int invocationCount = 0;
-
-    private FaultToleranceCommand command;
-
-    private ClassLoader contextClassLoader;
-
-    private final long delayCorrection;
-
-    private final int commandThreadPoolSize;
-
-    private final long threadWaitingPeriod;
-
-    private final long bulkheadTaskQueueingPeriod;
-
-    private CompletableFuture<?> taskQueued = new CompletableFuture<>();
-
-    /**
-     * Constructor.
-     *
-     * @param context The invocation context.
-     * @param introspector The method introspector.
-     */
-    public CommandRetrier(InvocationContext context, MethodIntrospector introspector) {
-        this.context = context;
-        this.introspector = introspector;
-        this.isAsynchronous = introspector.isAsynchronous();
-        this.method = context.getMethod();
-
-        // Init Helidon config params
-        Config config = ConfigProvider.getConfig();
-        this.delayCorrection = config.getOptionalValue(FT_DELAY_CORRECTION, Long.class)
-                .orElse(DEFAULT_DELAY_CORRECTION);
-        this.commandThreadPoolSize = config.getOptionalValue(FT_COMMAND_THREAD_POOL_SIZE, Integer.class)
-                .orElse(DEFAULT_COMMAND_THREAD_POOL_SIZE);
-        this.threadWaitingPeriod = config.getOptionalValue(FT_THREAD_WAITING_PERIOD, Long.class)
-                .orElse(DEFAULT_THREAD_WAITING_PERIOD);
-        this.bulkheadTaskQueueingPeriod = config.getOptionalValue(FT_BULKHEAD_TASK_QUEUEING_PERIOD, Long.class)
-                .orElse(DEFAULT_BULKHEAD_TASK_QUEUEING_PERIOD);
-
-        final Retry retry = introspector.getRetry();
-        if (retry != null) {
-            // Initial setting for retry policy
-            this.retryPolicy = new RetryPolicy<>()
-                                   .withMaxRetries(retry.maxRetries())
-                                   .withMaxDuration(Duration.of(retry.maxDuration(), retry.durationUnit()));
-            this.retryPolicy.handle(retry.retryOn());
-
-            // Set abortOn if defined
-            if (retry.abortOn().length > 0) {
-                this.retryPolicy.abortOn(retry.abortOn());
-            }
-
-            // Get delay and convert to nanos
-            long delay = TimeUtil.convertToNanos(retry.delay(), retry.delayUnit());
-
-            /*
-             * Apply delay correction to account for time spent in our code. This
-             * correction is necessary if user code measures intervals between
-             * calls that include time spent in Helidon. See TCK test {@link
-             * RetryTest#testRetryWithNoDelayAndJitter}. Failures may still occur
-             * on heavily loaded systems.
-             */
-            Function<Long, Long> correction =
-                    d -> Math.abs(d - TimeUtil.convertToNanos(delayCorrection, ChronoUnit.MILLIS));
-
-            // Processing for jitter and delay
-            if (retry.jitter() > 0) {
-                long jitter = TimeUtil.convertToNanos(retry.jitter(), retry.jitterDelayUnit());
-
-                // Need to compute a factor and adjust delay for Failsafe
-                double factor;
-                if (jitter > delay) {
-                    final long diff = jitter - delay;
-                    delay = delay + diff / 2;
-                    factor = 1.0;
-                } else {
-                    factor = ((double) jitter) / delay;
-                }
-                this.retryPolicy.withDelay(Duration.of(correction.apply(delay), ChronoUnit.NANOS));
-                this.retryPolicy.withJitter(factor);
-            } else if (retry.delay() > 0) {
-                this.retryPolicy.withDelay(Duration.of(correction.apply(delay), ChronoUnit.NANOS));
-            }
-        } else {
-            this.retryPolicy = new RetryPolicy<>().withMaxRetries(0);     // no retries
-        }
-    }
-
-    /**
-     * Get command thread pool size.
-     *
-     * @return Thread pool size.
-     */
-    int commandThreadPoolSize() {
-        return commandThreadPoolSize;
-    }
-
-    /**
-     * Get thread waiting period.
-     *
-     * @return Thread waiting period.
-     */
-    long threadWaitingPeriod() {
-        return threadWaitingPeriod;
-    }
-
-    FaultToleranceCommand getCommand() {
-        return command;
-    }
-
-    /**
-     * Retries running a command according to retry policy.
-     *
-     * @return Object returned by command.
-     * @throws Exception If something goes wrong.
-     */
-    public Object execute() throws Exception {
-        LOGGER.fine(() -> "Executing command with isAsynchronous = " + isAsynchronous);
-
-        FailsafeExecutor<Object> failsafe = prepareFailsafeExecutor();
-
-        try {
-            if (isAsynchronous) {
-                Scheduler scheduler = CommandScheduler.create(commandThreadPoolSize);
-                failsafe = failsafe.with(scheduler);
-
-                // Store context class loader to access config
-                contextClassLoader = Thread.currentThread().getContextClassLoader();
-
-                // Check CompletionStage first to process CompletableFuture here
-                if (introspector.isReturnType(CompletionStage.class)) {
-                    CompletionStage<?> completionStage = CommandCompletableFuture.create(
-                            failsafe.getStageAsync(() -> (CompletionStage<?>) retryExecute()),
-                            this::getCommand);
-                    awaitBulkheadAsyncTaskQueued();
-                    return completionStage;
-                }
-
-                // If not, it must be a subtype of Future
-                if (introspector.isReturnType(Future.class)) {
-                    Future<?> future = CommandCompletableFuture.create(
-                            failsafe.getAsync(() -> (Future<?>) retryExecute()),
-                            this::getCommand);
-                    awaitBulkheadAsyncTaskQueued();
-                    return future;
-                }
-
-                // Oops, something went wrong during validation
-                throw new InternalError("Validation failed, return type must be Future or CompletionStage");
-            } else {
-                return failsafe.get(this::retryExecute);
-            }
-        } catch (FailsafeException e) {
-            throw toException(e.getCause());
-        }
-    }
-
-    /**
-     * Set up the Failsafe executor. Add any fallback first, per Failsafe doc
-     * about "typical" policy composition
-     *
-     * @return Failsafe executor.
-     */
-    private FailsafeExecutor<Object> prepareFailsafeExecutor() {
-        List<Policy<Object>> policies = new ArrayList<>();
-        if (introspector.hasFallback()) {
-            CheckedFunction<ExecutionAttemptedEvent<?>, ?> fallbackFunction = event -> {
-                final CommandFallback fallback = new CommandFallback(context, introspector, event);
-                Object result = fallback.execute();
-                if (result instanceof CompletionStage<?>) {
-                    result = ((CompletionStage<?>) result).toCompletableFuture();
-                }
-                if (result instanceof Future<?>) {
-                    result = ((Future<?>) result).get();
-                }
-                return result;
-            };
-            policies.add(Fallback.of(fallbackFunction));
-        }
-        policies.add(retryPolicy);
-        return Failsafe.with(policies);
-    }
-
-    /**
-     * Creates a new command for each retry since Hystrix commands can only be
-     * executed once. Fallback method is not overridden here to ensure all
-     * retries are executed. If running in async mode, this method will execute
-     * on a different thread.
-     *
-     * @return Object returned by command.
-     */
-    private Object retryExecute() throws Exception {
-        // Config requires use of appropriate context class loader
-        if (contextClassLoader != null) {
-            Thread.currentThread().setContextClassLoader(contextClassLoader);
-        }
-
-        final String commandKey = createCommandKey();
-        command = new FaultToleranceCommand(this, commandKey, introspector, context,
-                contextClassLoader, taskQueued);
-
-        // Configure command before execution
-        introspector.getHystrixProperties()
-                .entrySet()
-                .forEach(entry -> setProperty(commandKey, entry.getKey(), entry.getValue()));
-
-        Object result;
-        try {
-            LOGGER.fine(() -> "About to execute command with key "
-                    + command.getCommandKey()
-                    + " on thread " + Thread.currentThread().getName());
-
-            // Execute task
-            invocationCount++;
-            updateMetricsBefore();
-            result = command.execute();
-            updateMetricsAfter(null);
-        } catch (ExceptionUtil.WrappedException e) {
-            Throwable cause = e.getCause();
-            if (cause instanceof HystrixRuntimeException) {
-                cause = cause.getCause();
-            }
-
-            updateMetricsAfter(cause);
-
-            if (cause instanceof TimeoutException) {
-                throw new org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException(cause);
-            }
-            if (isBulkheadRejection(cause)) {
-                throw new BulkheadException(cause);
-            }
-            if (isHystrixBreakerException(cause)) {
-                throw new CircuitBreakerOpenException(cause);
-            }
-            throw toException(cause);
-        }
-        return result;
-    }
-
-    /**
-     * A task can be queued on a bulkhead. When async and bulkheads are combined,
-     * we need to ensure that they get queued in the correct order before
-     * returning control back to the application. An exception thrown during
-     * queueing is processed in {@link FaultToleranceCommand#execute()}.
-     */
-    private void awaitBulkheadAsyncTaskQueued() {
-        if (introspector.hasBulkhead()) {
-            try {
-                taskQueued.get(bulkheadTaskQueueingPeriod, TimeUnit.MILLISECONDS);
-            } catch (Exception e) {
-                LOGGER.info(() -> "Bulkhead async task queueing exception " + e);
-            }
-        }
-    }
-
-    /**
-     * Update metrics before method is called.
-     */
-    private void updateMetricsBefore() {
-        if (isFaultToleranceMetricsEnabled()) {
-            if (introspector.hasRetry() && invocationCount > 1) {
-                getCounter(method, RETRY_RETRIES_TOTAL).inc();
-            }
-        }
-    }
-
-    /**
-     * Update metrics after method is called and depending on outcome.
-     *
-     * @param cause Exception cause or {@code null} if execution successful.
-     */
-    private void updateMetricsAfter(Throwable cause) {
-        if (!isFaultToleranceMetricsEnabled()) {
-            return;
-        }
-
-        // Special logic for methods with retries
-        if (introspector.hasRetry()) {
-            final Retry retry = introspector.getRetry();
-            boolean firstInvocation = (invocationCount == 1);
-
-            if (cause == null) {
-                getCounter(method, INVOCATIONS_TOTAL).inc();
-                getCounter(method, firstInvocation
-                                   ? RETRY_CALLS_SUCCEEDED_NOT_RETRIED_TOTAL
-                                   : RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL).inc();
-            } else if (retry.maxRetries() == invocationCount - 1) {
-                getCounter(method, RETRY_CALLS_FAILED_TOTAL).inc();
-                getCounter(method, INVOCATIONS_FAILED_TOTAL).inc();
-                getCounter(method, INVOCATIONS_TOTAL).inc();
-            }
-        } else {
-            // Global method counters
-            getCounter(method, INVOCATIONS_TOTAL).inc();
-            if (cause != null) {
-                getCounter(method, INVOCATIONS_FAILED_TOTAL).inc();
-            }
-        }
-
-        // Timeout
-        if (introspector.hasTimeout()) {
-            getHistogram(method, TIMEOUT_EXECUTION_DURATION)
-                                 .update(command.getExecutionTime());
-            getCounter(method, cause instanceof TimeoutException
-                               ? TIMEOUT_CALLS_TIMED_OUT_TOTAL
-                               : TIMEOUT_CALLS_NOT_TIMED_OUT_TOTAL).inc();
-        }
-
-        // Bulkhead
-        if (introspector.hasBulkhead()) {
-            boolean bulkheadRejection = isBulkheadRejection(cause);
-            if (!bulkheadRejection) {
-                getHistogram(method, BULKHEAD_EXECUTION_DURATION).update(command.getExecutionTime());
-            }
-            getCounter(method, bulkheadRejection ? BULKHEAD_CALLS_REJECTED_TOTAL
-                    : BULKHEAD_CALLS_ACCEPTED_TOTAL).inc();
-        }
-    }
-
-    /**
-     * Returns a key for a command. Keys are specific to the pair of instance (target)
-     * and the method being called.
-     *
-     * @return A command key.
-     */
-    private String createCommandKey() {
-        return method.getName() + Objects.hash(context.getTarget(), context.getMethod().hashCode());
-    }
-
-    /**
-     * Sets a Hystrix property on a command.
-     *
-     * @param commandKey Command key.
-     * @param key Property key.
-     * @param value Property value.
-     */
-    private void setProperty(String commandKey, String key, Object value) {
-        final String actualKey = String.format("hystrix.command.%s.%s", commandKey, key);
-        synchronized (ConfigurationManager.getConfigInstance()) {
-            final AbstractConfiguration configManager = ConfigurationManager.getConfigInstance();
-            if (configManager.getProperty(actualKey) == null) {
-                configManager.setProperty(actualKey, value);
-            }
-        }
-    }
-
-    /**
-     * Hystrix throws a {@code RuntimeException}, so we need to check
-     * the message to determine if it is a breaker exception.
-     *
-     * @param t Throwable to check.
-     * @return Outcome of test.
-     */
-    private static boolean isHystrixBreakerException(Throwable t) {
-        return t instanceof RuntimeException && t.getMessage().contains("Hystrix "
-                + "circuit short-circuited and is OPEN");
-    }
-
-    /**
-     * Checks if the parameter is a bulkhead exception. Note that Hystrix with semaphore
-     * isolation may throw a {@code RuntimeException}, so we need to check the message
-     * to determine if it is a semaphore exception.
-     *
-     * @param t Throwable to check.
-     * @return Outcome of test.
-     */
-    private static boolean isBulkheadRejection(Throwable t) {
-        return t instanceof RejectedExecutionException
-                || t instanceof RuntimeException && t.getMessage().contains("could "
-                + "not acquire a semaphore for execution");
-    }
-}

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandScheduler.java

@@ -1,82 +0,0 @@
-/*
- * Copyright (c) 2018, 2020 Oracle and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.helidon.microprofile.faulttolerance;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-
-import io.helidon.common.configurable.ScheduledThreadPoolSupplier;
-
-import net.jodah.failsafe.util.concurrent.Scheduler;
-
-/**
- * Class CommandScheduler.
- */
-public class CommandScheduler implements Scheduler {
-
-    private static final String THREAD_NAME_PREFIX = "helidon-ft-async-";
-
-    private static CommandScheduler instance;
-
-    private final ScheduledThreadPoolSupplier poolSupplier;
-
-    private CommandScheduler(ScheduledThreadPoolSupplier poolSupplier) {
-        this.poolSupplier = poolSupplier;
-    }
-
-    /**
-     * If no command scheduler exists, creates one using default values.
-     * The created command scheduler uses daemon threads, so the JVM shuts-down if these are the only ones running.
-     *
-     * @param threadPoolSize Size of thread pool for async commands.
-     * @return Existing scheduler or newly created one.
-     */
-    public static synchronized CommandScheduler create(int threadPoolSize) {
-        if (instance == null) {
-            instance = new CommandScheduler(ScheduledThreadPoolSupplier.builder()
-                    .daemon(true)
-                    .threadNamePrefix(THREAD_NAME_PREFIX)
-                    .corePoolSize(threadPoolSize)
-                    .prestart(false)
-                    .build());
-        }
-        return instance;
-    }
-
-    /**
-     * Returns underlying pool supplier.
-     *
-     * @return The pool supplier.
-     */
-    ScheduledThreadPoolSupplier poolSupplier() {
-        return poolSupplier;
-    }
-
-    /**
-     * Schedules a task using an executor.
-     *
-     * @param callable The callable.
-     * @param delay Delay before scheduling task.
-     * @param unit Unite of delay.
-     * @return Future to track task execution.
-     */
-    @Override
-    public ScheduledFuture<?> schedule(Callable<?> callable, long delay, TimeUnit unit) {
-        return poolSupplier.get().schedule(callable, delay, unit);
-    }
-}

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ExceptionUtil.java

@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.helidon.microprofile.faulttolerance;
-
-import com.netflix.hystrix.exception.HystrixRuntimeException;
-
-/**
- * Class ExceptionUtil.
- */
-public class ExceptionUtil {
-
-    /**
-     * Exception used internally to propagate other exceptions.
-     */
-    static class WrappedException extends RuntimeException {
-        WrappedException(Throwable t) {
-            super(t);
-        }
-    }
-
-    /**
-     * Wrap throwable into {@code Exception}.
-     *
-     * @param throwable The throwable.
-     * @return A {@code RuntimeException}.
-     */
-    public static Exception toException(Throwable throwable) {
-        return throwable instanceof Exception ? (Exception) throwable
-                : new RuntimeException(throwable);
-    }
-
-    /**
-     * Wrap throwable into {@code RuntimeException}.
-     *
-     * @param throwable The throwable.
-     * @return A {@code RuntimeException}.
-     */
-    public static WrappedException toWrappedException(Throwable throwable) {
-        return throwable instanceof WrappedException ? (WrappedException) throwable
-                : new WrappedException(throwable);
-    }
-
-    /**
-     * Unwrap an throwable wrapped by {@code HystrixRuntimeException}.
-     *
-     * @param throwable Throwable to unwrap.
-     * @return Unwrapped throwable.
-     */
-    public static Throwable unwrapHystrix(Throwable throwable) {
-        return throwable instanceof HystrixRuntimeException ? throwable.getCause() : throwable;
-    }
-
-    private ExceptionUtil() {
-    }
-}

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FallbackAntn.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,8 +17,6 @@
 package io.helidon.microprofile.faulttolerance;
 
 import java.lang.reflect.Method;
-import java.util.concurrent.CompletionStage;
-import java.util.concurrent.Future;
 
 import org.eclipse.microprofile.faulttolerance.ExecutionContext;
 import org.eclipse.microprofile.faulttolerance.Fallback;
@@ -58,11 +56,11 @@ public class FallbackAntn extends MethodAntn implements Fallback {
                 final Method fallbackMethod = JavaMethodFinder.findMethod(method.getDeclaringClass(),
                         methodName,
                         method.getGenericParameterTypes());
-                if (!fallbackMethod.getReturnType().isAssignableFrom(method.getReturnType())
-                        && !Future.class.isAssignableFrom(method.getReturnType())
-                        && !CompletionStage.class.isAssignableFrom(method.getReturnType())) {        // async
-                    throw new FaultToleranceDefinitionException("Fallback method return type "
-                            + "is invalid: " + fallbackMethod.getReturnType());
+                if (!method.getReturnType().isAssignableFrom(fallbackMethod.getReturnType())) {
+                    throw new FaultToleranceDefinitionException("Fallback method " + fallbackMethod.getName()
+                            + " in class " + fallbackMethod.getDeclaringClass().getSimpleName()
+                            + " incompatible return type " + fallbackMethod.getReturnType()
+                            + " with " + method.getReturnType());
                 }
             } catch (NoSuchMethodException e) {
                 throw new FaultToleranceDefinitionException(e);
@@ -103,4 +101,18 @@ public class FallbackAntn extends MethodAntn implements Fallback {
         final String override = getParamOverride("fallbackMethod", lookupResult.getType());
         return override != null ? override : lookupResult.getAnnotation().fallbackMethod();
     }
+
+    @Override
+    public Class<? extends Throwable>[] applyOn() {
+        LookupResult<Fallback> lookupResult = lookupAnnotation(Fallback.class);
+        final String override = getParamOverride("applyOn", lookupResult.getType());
+        return override != null ? parseThrowableArray(override) : lookupResult.getAnnotation().applyOn();
+    }
+
+    @Override
+    public Class<? extends Throwable>[] skipOn() {
+        LookupResult<Fallback> lookupResult = lookupAnnotation(Fallback.class);
+        final String override = getParamOverride("skipOn", lookupResult.getType());
+        return override != null ? parseThrowableArray(override) : lookupResult.getAnnotation().skipOn();
+    }
 }

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceCommand.java

@@ -1,514 +0,0 @@
-/*
- * Copyright (c) 2018, 2020 Oracle and/or its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.helidon.microprofile.faulttolerance;
-
-import java.lang.reflect.Method;
-import java.util.Arrays;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.logging.Logger;
-
-import javax.enterprise.context.control.RequestContextController;
-import javax.enterprise.inject.spi.CDI;
-import javax.interceptor.InvocationContext;
-
-import io.helidon.common.context.Context;
-import io.helidon.common.context.Contexts;
-
-import com.netflix.hystrix.HystrixCommand;
-import com.netflix.hystrix.HystrixCommandGroupKey;
-import com.netflix.hystrix.HystrixCommandKey;
-import com.netflix.hystrix.HystrixCommandProperties;
-import com.netflix.hystrix.HystrixThreadPoolKey;
-import com.netflix.hystrix.HystrixThreadPoolProperties;
-import org.eclipse.microprofile.metrics.Histogram;
-import org.glassfish.jersey.process.internal.RequestContext;
-import org.glassfish.jersey.process.internal.RequestScope;
-
-import static com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy.SEMAPHORE;
-import static com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy.THREAD;
-import static io.helidon.microprofile.faulttolerance.CircuitBreakerHelper.State;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceExtension.isFaultToleranceMetricsEnabled;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_PREVENTED_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_SUCCEEDED_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CLOSED_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_HALF_OPEN_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_OPENED_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_OPEN_TOTAL;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_CONCURRENT_EXECUTIONS;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_WAITING_DURATION;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_WAITING_QUEUE_POPULATION;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.METRIC_NAME_TEMPLATE;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getCounter;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getHistogram;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.registerGauge;
-import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.registerHistogram;
-
-/**
- * Class FaultToleranceCommand.
- */
-public class FaultToleranceCommand extends HystrixCommand<Object> {
-    private static final Logger LOGGER = Logger.getLogger(FaultToleranceCommand.class.getName());
-
-    static final String HELIDON_MICROPROFILE_FAULTTOLERANCE = "io.helidon.microprofile.faulttolerance";
-
-    private final String commandKey;
-
-    private final MethodIntrospector introspector;
-
-    private final InvocationContext context;
-
-    private long executionTime = -1L;
-
-    private CircuitBreakerHelper breakerHelper;
-
-    private BulkheadHelper bulkheadHelper;
-
-    private long queuedNanos = -1L;
-
-    private Thread runThread;
-
-    private ClassLoader contextClassLoader;
-
-    private final long threadWaitingPeriod;
-
-    /**
-     * Helidon context in which to run business method.
-     */
-    private Context helidonContext;
-
-    private CompletableFuture<?> taskQueued;
-
-    private RequestScope requestScope;
-
-    private RequestContextController requestController;
-
-    private RequestContext requestContext;
-
-    /**
-     * Constructor. Specify a thread pool key if a {@code @Bulkhead} is specified
-     * on the method. A unique thread pool key will enable setting limits for this
-     * command only based on the {@code Bulkhead} properties.
-     *
-     * @param commandRetrier The command retrier associated with this command.
-     * @param commandKey The command key.
-     * @param introspector The method introspector.
-     * @param context CDI invocation context.
-     * @param contextClassLoader Context class loader or {@code null} if not available.
-     * @param taskQueued Future completed when task has been queued.
-     */
-    public FaultToleranceCommand(CommandRetrier commandRetrier, String commandKey,
-                                 MethodIntrospector introspector,
-                                 InvocationContext context, ClassLoader contextClassLoader,
-                                 CompletableFuture<?> taskQueued) {
-        super(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey(HELIDON_MICROPROFILE_FAULTTOLERANCE))
-                .andCommandKey(
-                        HystrixCommandKey.Factory.asKey(commandKey))
-                .andCommandPropertiesDefaults(
-                        HystrixCommandProperties.Setter()
-                                .withFallbackEnabled(false)
-                                .withExecutionIsolationStrategy(introspector.hasBulkhead()
-                                        && !introspector.isAsynchronous() ? SEMAPHORE : THREAD)
-                                .withExecutionIsolationThreadInterruptOnFutureCancel(true)
-                                .withExecutionIsolationThreadInterruptOnTimeout(true)
-                                .withExecutionTimeoutEnabled(false))
-                .andThreadPoolKey(
-                        introspector.hasBulkhead()
-                                ? HystrixThreadPoolKey.Factory.asKey(commandKey)
-                                : null)
-                .andThreadPoolPropertiesDefaults(
-                        HystrixThreadPoolProperties.Setter()
-                                .withCoreSize(
-                                        introspector.hasBulkhead()
-                                                ? introspector.getBulkhead().value()
-                                                : commandRetrier.commandThreadPoolSize())
-                                .withMaximumSize(
-                                        introspector.hasBulkhead()
-                                                ? introspector.getBulkhead().value()
-                                                : commandRetrier.commandThreadPoolSize())
-                                .withMaxQueueSize(
-                                        introspector.hasBulkhead() && introspector.isAsynchronous()
-                                                ? introspector.getBulkhead().waitingTaskQueue()
-                                                : -1)
-                                .withQueueSizeRejectionThreshold(
-                                        introspector.hasBulkhead() && introspector.isAsynchronous()
-                                                ? introspector.getBulkhead().waitingTaskQueue()
-                                                : -1)));
-        this.commandKey = commandKey;
-        this.introspector = introspector;
-        this.context = context;
-        this.contextClassLoader = contextClassLoader;
-        this.threadWaitingPeriod = commandRetrier.threadWaitingPeriod();
-        this.taskQueued = taskQueued;
-
-        // Gather information about current request scope if active
-        try {
-            requestScope = CDI.current().select(RequestScope.class).get();
-            requestContext = requestScope.current();
-            requestController = CDI.current().select(RequestContextController.class).get();
-        } catch (Exception e) {
-            requestScope = null;
-            LOGGER.info(() -> "Request context not active for command " + getCommandKey()
-                    + " on thread " + Thread.currentThread().getName());
-        }
-
-        // Special initialization for methods with breakers
-        if (introspector.hasCircuitBreaker()) {
-            this.breakerHelper = new CircuitBreakerHelper(this, introspector.getCircuitBreaker());
-
-            // Register gauges for this method
-            if (isFaultToleranceMetricsEnabled()) {
-                registerGauge(introspector.getMethod(),
-                        BREAKER_OPEN_TOTAL,
-                        "Amount of time the circuit breaker has spent in open state",
-                        () -> breakerHelper.getInStateNanos(State.OPEN_MP));
-                registerGauge(introspector.getMethod(),
-                        BREAKER_HALF_OPEN_TOTAL,
-                        "Amount of time the circuit breaker has spent in half-open state",
-                        () -> breakerHelper.getInStateNanos(State.HALF_OPEN_MP));
-                registerGauge(introspector.getMethod(),
-                        BREAKER_CLOSED_TOTAL,
-                        "Amount of time the circuit breaker has spent in closed state",
-                        () -> breakerHelper.getInStateNanos(State.CLOSED_MP));
-            }
-        }
-
-        if (introspector.hasBulkhead()) {
-            bulkheadHelper = new BulkheadHelper(commandKey, introspector.getBulkhead());
-
-            if (isFaultToleranceMetricsEnabled()) {
-                // Record nanos to update metrics later
-                queuedNanos = System.nanoTime();
-
-                // Register gauges for this method
-                registerGauge(introspector.getMethod(),
-                        BULKHEAD_CONCURRENT_EXECUTIONS,
-                        "Number of currently running executions",
-                        () -> (long) bulkheadHelper.runningInvocations());
-                if (introspector.isAsynchronous()) {
-                    registerGauge(introspector.getMethod(),
-                            BULKHEAD_WAITING_QUEUE_POPULATION,
-                            "Number of executions currently waiting in the queue",
-                            () -> (long) bulkheadHelper.waitingInvocations());
-                }
-            }
-        }
-    }
-
-    /**
-     * Get command's execution time in nanos.
-     *
-     * @return Execution time in nanos.
-     * @throws IllegalStateException If called before command is executed.
-     */
-    long getExecutionTime() {
-        if (executionTime == -1L) {
-            throw new IllegalStateException("Command has not been executed yet");
-        }
-        return executionTime;
-    }
-
-    BulkheadHelper getBulkheadHelper() {
-        return bulkheadHelper;
-    }
-
-    /**
-     * Code to run as part of this command. Called from superclass.
-     *
-     * @return Result of command.
-     * @throws Exception If an error occurs.
-     */
-    @Override
-    public Object run() throws Exception {
-        // Config requires use of appropriate context class loader
-        if (contextClassLoader != null) {
-            Thread.currentThread().setContextClassLoader(contextClassLoader);
-        }
-
-        if (introspector.hasBulkhead()) {
-            bulkheadHelper.markAsRunning(this);
-
-            if (isFaultToleranceMetricsEnabled()) {
-                // Register and update waiting time histogram
-                if (introspector.isAsynchronous() && queuedNanos != -1L) {
-                    Method method = introspector.getMethod();
-                    Histogram histogram = getHistogram(method, BULKHEAD_WAITING_DURATION);
-                    if (histogram == null) {
-                        registerHistogram(
-                                String.format(METRIC_NAME_TEMPLATE,
-                                        method.getDeclaringClass().getName(),
-                                        method.getName(),
-                                        BULKHEAD_WAITING_DURATION),
-                                "Histogram of the time executions spend waiting in the queue");
-                        histogram = getHistogram(method, BULKHEAD_WAITING_DURATION);
-                    }
-                    histogram.update(System.nanoTime() - queuedNanos);
-                }
-            }
-        }
-
-        // Create callable to invoke method
-        Callable<Object> callMethod = () -> {
-            try {
-                runThread = Thread.currentThread();
-                return Contexts.runInContextWithThrow(helidonContext, context::proceed);
-            } finally {
-                if (introspector.hasBulkhead()) {
-                    bulkheadHelper.markAsNotRunning(this);
-                }
-            }
-        };
-
-        // Call method in request scope if active
-        if (requestScope != null) {
-            return requestScope.runInScope(requestContext, (Callable<Object>) (() -> {
-                try {
-                    requestController.activate();
-                    return callMethod.call();
-                } finally {
-                    requestController.deactivate();
-                }
-            }));
-        } else {
-            return callMethod.call();
-        }
-    }
-
-    /**
-     * Executes this command returning a result or throwing an exception.
-     *
-     * @return The result.
-     * @throws RuntimeException If something goes wrong.
-     */
-    @Override
-    public Object execute() {
-        this.helidonContext = Contexts.context().orElseGet(Context::create);
-        boolean lockRemoved = false;
-
-        // Get lock and check breaker delay
-        if (introspector.hasCircuitBreaker()) {
-            try {
-                breakerHelper.lock();
-                // OPEN_MP -> HALF_OPEN_MP
-                if (breakerHelper.getState() == State.OPEN_MP) {
-                    long delayNanos = TimeUtil.convertToNanos(introspector.getCircuitBreaker().delay(),
-                            introspector.getCircuitBreaker().delayUnit());
-                    if (breakerHelper.getCurrentStateNanos() > delayNanos) {
-                        breakerHelper.setState(State.HALF_OPEN_MP);
-                    }
-                }
-            } finally {
-                breakerHelper.unlock();
-            }
-
-            logCircuitBreakerState("Enter");
-        }
-
-        // Record state of breaker
-        boolean wasBreakerOpen = isCircuitBreakerOpen();
-
-        // Track invocation in a bulkhead
-        if (introspector.hasBulkhead()) {
-            bulkheadHelper.trackInvocation(this);
-        }
-
-        // Execute command
-        Object result = null;
-        Future<Object> future = null;
-        Throwable throwable = null;
-        long startNanos = System.nanoTime();
-        try {
-            // Queue the task
-            future = super.queue();
-
-            // Notify successful queueing of task
-            taskQueued.complete(null);
-
-            // Execute and get result from task
-            result = future.get();
-        } catch (Exception e) {
-            // Notify exception during task queueing
-            taskQueued.completeExceptionally(e);
-
-            if (e instanceof ExecutionException) {
-                waitForThreadToComplete();
-            }
-            if (e instanceof InterruptedException) {
-                future.cancel(true);
-            }
-            throwable = decomposeException(e);
-        }
-
-        executionTime = System.nanoTime() - startNanos;
-        boolean hasFailed = (throwable != null);
-
-        if (introspector.hasCircuitBreaker()) {
-            try {
-                breakerHelper.lock();
-
-                // Keep track of failure ratios
-                breakerHelper.pushResult(throwable == null);
-
-                // Query breaker states
-                boolean breakerOpening = false;
-                boolean isClosedNow = !wasBreakerOpen;
-
-                /*
-                 * Special logic for MP circuit breakers to support failOn. If not a
-                 * throwable to fail on, restore underlying breaker and return.
-                 */
-                if (hasFailed) {
-                    final Throwable unwrappedThrowable = ExceptionUtil.unwrapHystrix(throwable);
-                    Class<? extends Throwable>[] throwableClasses = introspector.getCircuitBreaker().failOn();
-                    boolean failOn = Arrays.asList(throwableClasses)
-                            .stream()
-                            .anyMatch(c -> c.isAssignableFrom(unwrappedThrowable.getClass()));
-                    if (!failOn) {
-                        // If underlying circuit breaker is not open, this counts as successful
-                        // run since it failed on an exception not listed in failOn.
-                        updateMetricsAfter(breakerHelper.getState() != State.OPEN_MP ? null : throwable,
-                                wasBreakerOpen, isClosedNow, breakerOpening);
-                        logCircuitBreakerState("Exit 1");
-                        throw ExceptionUtil.toWrappedException(throwable);
-                    }
-                }
-
-                // CLOSED_MP -> OPEN_MP
-                if (breakerHelper.getState() == State.CLOSED_MP) {
-                    double failureRatio = breakerHelper.getFailureRatio();
-                    if (failureRatio >= introspector.getCircuitBreaker().failureRatio()) {
-                        breakerHelper.setState(State.OPEN_MP);
-                        breakerOpening = true;
-                    }
-                }
-
-                // HALF_OPEN_MP -> OPEN_MP
-                if (hasFailed) {
-                    if (breakerHelper.getState() == State.HALF_OPEN_MP) {
-                        breakerHelper.setState(State.OPEN_MP);
-                    }
-                    updateMetricsAfter(throwable, wasBreakerOpen, isClosedNow, breakerOpening);
-                    logCircuitBreakerState("Exit 2");
-                    throw ExceptionUtil.toWrappedException(throwable);
-                }
-
-                // Otherwise, increment success count
-                breakerHelper.incSuccessCount();
-
-                // HALF_OPEN_MP -> CLOSED_MP
-                if (breakerHelper.getState() == State.HALF_OPEN_MP) {
-                    if (breakerHelper.getSuccessCount() == introspector.getCircuitBreaker().successThreshold()) {
-                        breakerHelper.setState(State.CLOSED_MP);
-                        breakerHelper.resetCommandData();
-                        lockRemoved = true;
-                        isClosedNow = true;
-                    }
-                }
-
-                updateMetricsAfter(throwable, wasBreakerOpen, isClosedNow, breakerOpening);
-            } finally {
-                if (!lockRemoved) {
-                    breakerHelper.unlock();
-                }
-            }
-        }
-
-        // Untrack invocation in a bulkhead
-        if (introspector.hasBulkhead()) {
-            bulkheadHelper.untrackInvocation(this);
-        }
-
-        // Display circuit breaker state at exit
-        logCircuitBreakerState("Exit 3");
-
-        // Outcome of execution
-        if (throwable != null) {
-            throw ExceptionUtil.toWrappedException(throwable);
-        } else {
-            return result;
-        }
-    }
-
-    private void updateMetricsAfter(Throwable throwable, boolean wasBreakerOpen, boolean isClosedNow,
-                                    boolean breakerWillOpen) {
-        if (!isFaultToleranceMetricsEnabled()) {
-            return;
-        }
-
-        assert introspector.hasCircuitBreaker();
-        Method method = introspector.getMethod();
-
-        if (throwable == null) {
-            // If no errors increment success counter
-            getCounter(method, BREAKER_CALLS_SUCCEEDED_TOTAL).inc();
-        } else if (!wasBreakerOpen) {
-            // If error and breaker was closed, increment failed counter
-            getCounter(method, BREAKER_CALLS_FAILED_TOTAL).inc();
-            // If it will open, increment counter
-            if (breakerWillOpen) {
-                getCounter(method, BREAKER_OPENED_TOTAL).inc();
-            }
-        }
-        // If breaker was open and still is, increment prevented counter
-        if (wasBreakerOpen && !isClosedNow) {
-            getCounter(method, BREAKER_CALLS_PREVENTED_TOTAL).inc();
-        }
-    }
-
-    /**
-     * Logs circuit breaker state, if one is present.
-     *
-     * @param preamble Message preamble.
-     */
-    private void logCircuitBreakerState(String preamble) {
-        if (introspector.hasCircuitBreaker()) {
-            String hystrixState = isCircuitBreakerOpen() ? "OPEN" : "CLOSED";
-            LOGGER.fine(() -> preamble + ": breaker for " + getCommandKey() + " in state "
-                    + breakerHelper.getState() + " (Hystrix: " + hystrixState
-                    + " Thread:" + Thread.currentThread().getName() + ")");
-        }
-    }
-
-    /**
-     * <p>After a timeout expires, Hystrix can report an {@link ExecutionException}
-     * when a thread has been interrupted but it is still running (e.g. while in a
-     * busy loop). Hystrix makes this possible by using another thread to monitor
-     * the command's thread.</p>
-     *
-     * <p>According to the FT spec, the thread may continue to run, so here
-     * we give it a chance to do that before completing the execution of the
-     * command. For more information see TCK test {@code
-     * TimeoutUninterruptableTest::testTimeout}.</p>
-     */
-    private void waitForThreadToComplete() {
-        if (!introspector.isAsynchronous() && runThread != null && runThread.isInterrupted()) {
-            try {
-                int waitTime = 250;
-                while (runThread.getState() == Thread.State.RUNNABLE && waitTime <= threadWaitingPeriod) {
-                    LOGGER.fine(() -> "Waiting for completion of thread " + runThread);
-                    Thread.sleep(waitTime);
-                    waitTime += 250;
-                }
-            } catch (InterruptedException e) {
-                // Falls through
-            }
-        }
-    }
-}

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java

@@ -42,6 +42,10 @@ import javax.enterprise.inject.spi.ProcessSyntheticBean;
 import javax.enterprise.util.AnnotationLiteral;
 import javax.inject.Inject;
 
+import io.helidon.common.configurable.ScheduledThreadPoolSupplier;
+import io.helidon.common.configurable.ThreadPoolSupplier;
+import io.helidon.faulttolerance.FaultTolerance;
+
 import org.eclipse.microprofile.config.Config;
 import org.eclipse.microprofile.config.ConfigProvider;
 import org.eclipse.microprofile.faulttolerance.Asynchronous;
@@ -221,11 +225,11 @@ public class FaultToleranceExtension implements Extension {
     }
 
     /**
-     * Registers metrics for all FT methods.
+     * Registers metrics for all FT methods and init executors.
      *
      * @param validation Event information.
      */
-    void registerFaultToleranceMetrics(@Observes AfterDeploymentValidation validation) {
+    void registerMetricsAndInitExecutors(@Observes AfterDeploymentValidation validation) {
         if (FaultToleranceMetrics.enabled()) {
             getRegisteredMethods().stream().forEach(beanMethod -> {
                 final Method method = beanMethod.method();
@@ -260,6 +264,19 @@ public class FaultToleranceExtension implements Extension {
                 }
             });
         }
+
+        // Initialize executors for MP FT - default size of 16
+        io.helidon.config.Config config = io.helidon.config.Config.create();
+        FaultTolerance.scheduledExecutor(ScheduledThreadPoolSupplier.builder()
+                .threadNamePrefix("ft-mp-schedule-")
+                .corePoolSize(16)
+                .config(config.get("scheduled-executor"))
+                .build());
+        FaultTolerance.executor(ThreadPoolSupplier.builder()
+                .threadNamePrefix("ft-mp-")
+                .corePoolSize(16)
+                .config(config.get("executor"))
+                .build());
     }
 
     /**

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceParameter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -53,7 +53,8 @@ class FaultToleranceParameter {
      */
     private static String getProperty(String name) {
         try {
-            String value = ConfigProvider.getConfig().getValue(name, String.class);
+            ClassLoader ccl = Thread.currentThread().getContextClassLoader();
+            String value = ConfigProvider.getConfig(ccl).getValue(name, String.class);
             LOGGER.fine(() -> "Found config property '" + name + "' value '" + value + "'");
             return value;
         } catch (NoSuchElementException e) {

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FtSupplier.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.helidon.microprofile.faulttolerance;
+
+/**
+ * A special supplier that can also throw a {@link java.lang.Throwable}.
+ *
+ * @param <T> Type provided by this supplier.
+ */
+@FunctionalInterface
+interface FtSupplier<T> {
+    T get() throws Throwable;
+}

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/InvokerAsyncException.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.helidon.microprofile.faulttolerance;
+
+/**
+ * Wraps a {@code Throwable} thrown by an async call. It is necessary to
+ * distinguish it from exceptions thrown by the intercepted method.
+ */
+class InvokerAsyncException extends Exception {
+
+    InvokerAsyncException(Throwable cause) {
+        super(cause);
+    }
+}

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodIntrospector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,8 +18,6 @@ package io.helidon.microprofile.faulttolerance;
 
 import java.lang.annotation.Annotation;
 import java.lang.reflect.Method;
-import java.util.HashMap;
-import java.util.Map;
 
 import io.helidon.microprofile.faulttolerance.MethodAntn.LookupResult;
 
@@ -42,15 +40,15 @@ class MethodIntrospector {
 
     private final Class<?> beanClass;
 
-    private Retry retry;
+    private final Retry retry;
 
-    private Fallback fallback;
+    private final Fallback fallback;
 
-    private CircuitBreaker circuitBreaker;
+    private final CircuitBreaker circuitBreaker;
 
-    private Timeout timeout;
+    private final Timeout timeout;
 
-    private Bulkhead bulkhead;
+    private final Bulkhead bulkhead;
 
     /**
      * Constructor.
@@ -69,7 +67,7 @@ class MethodIntrospector {
         this.fallback = isAnnotationEnabled(Fallback.class) ? new FallbackAntn(beanClass, method) : null;
     }
 
-    Method getMethod() {
+    Method method() {
         return method;
     }
 
@@ -152,41 +150,6 @@ class MethodIntrospector {
         return bulkhead;
     }
 
-    /**
-     * Returns a collection of Hystrix properties needed to configure
-     * commands. These properties are derived from the set of annotations
-     * found on a method or its class.
-     *
-     * @return The collection of Hystrix properties.
-     */
-    Map<String, Object> getHystrixProperties() {
-        final HashMap<String, Object> result = new HashMap<>();
-
-        // Use semaphores for async and bulkhead
-        if (!isAsynchronous() && hasBulkhead()) {
-            result.put("execution.isolation.semaphore.maxConcurrentRequests", bulkhead.value());
-        }
-
-        // Circuit breakers
-        result.put("circuitBreaker.enabled", hasCircuitBreaker());
-        if (hasCircuitBreaker()) {
-            // We are implementing this logic internally, so set to high values
-            result.put("circuitBreaker.requestVolumeThreshold", Integer.MAX_VALUE);
-            result.put("circuitBreaker.errorThresholdPercentage", 100);
-            result.put("circuitBreaker.sleepWindowInMilliseconds", Long.MAX_VALUE);
-        }
-
-        // Timeouts
-        result.put("execution.timeout.enabled", hasTimeout());
-        if (hasTimeout()) {
-            final Timeout timeout = getTimeout();
-            result.put("execution.isolation.thread.timeoutInMilliseconds",
-                       TimeUtil.convertToMillis(timeout.value(), timeout.unit()));
-        }
-
-        return result;
-    }
-
     /**
      * Determines if annotation type is present and enabled.
      *
@@ -206,19 +169,19 @@ class MethodIntrospector {
         value = getParameter(method.getDeclaringClass().getName(), method.getName(),
                 annotationType, "enabled");
         if (value != null) {
-            return Boolean.valueOf(value);
+            return Boolean.parseBoolean(value);
         }
 
         // Check if property defined at class level
         value = getParameter(method.getDeclaringClass().getName(), annotationType, "enabled");
         if (value != null) {
-            return Boolean.valueOf(value);
+            return Boolean.parseBoolean(value);
         }
 
         // Check if property defined at global level
         value = getParameter(annotationType, "enabled");
         if (value != null) {
-            return Boolean.valueOf(value);
+            return Boolean.parseBoolean(value);
         }
 
         // Default is enabled

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java

@@ -0,0 +1,791 @@
+/*
+ * Copyright (c) 2020 Oracle and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.helidon.microprofile.faulttolerance;
+
+import java.lang.reflect.Method;
+import java.time.Duration;
+import java.util.Objects;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+
+import javax.enterprise.context.control.RequestContextController;
+import javax.enterprise.inject.spi.CDI;
+import javax.interceptor.InvocationContext;
+
+import io.helidon.common.context.Context;
+import io.helidon.common.context.Contexts;
+import io.helidon.common.reactive.Single;
+import io.helidon.faulttolerance.Async;
+import io.helidon.faulttolerance.Bulkhead;
+import io.helidon.faulttolerance.CircuitBreaker;
+import io.helidon.faulttolerance.CircuitBreaker.State;
+import io.helidon.faulttolerance.Fallback;
+import io.helidon.faulttolerance.FaultTolerance;
+import io.helidon.faulttolerance.FtHandlerTyped;
+import io.helidon.faulttolerance.Retry;
+import io.helidon.faulttolerance.Timeout;
+
+import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException;
+import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException;
+import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException;
+import org.eclipse.microprofile.metrics.Counter;
+import org.glassfish.jersey.process.internal.RequestContext;
+import org.glassfish.jersey.process.internal.RequestScope;
+
+import static io.helidon.microprofile.faulttolerance.FaultToleranceExtension.isFaultToleranceMetricsEnabled;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_PREVENTED_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_SUCCEEDED_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CLOSED_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_HALF_OPEN_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_OPENED_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_OPEN_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_CALLS_ACCEPTED_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_CALLS_REJECTED_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_CONCURRENT_EXECUTIONS;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_EXECUTION_DURATION;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_WAITING_DURATION;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_WAITING_QUEUE_POPULATION;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.INVOCATIONS_FAILED_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.INVOCATIONS_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.METRIC_NAME_TEMPLATE;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_FAILED_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_SUCCEEDED_NOT_RETRIED_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_RETRIES_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.TIMEOUT_CALLS_NOT_TIMED_OUT_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.TIMEOUT_CALLS_TIMED_OUT_TOTAL;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.TIMEOUT_EXECUTION_DURATION;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getCounter;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getHistogram;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.registerGauge;
+import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.registerHistogram;
+import static io.helidon.microprofile.faulttolerance.ThrowableMapper.map;
+import static io.helidon.microprofile.faulttolerance.ThrowableMapper.mapTypes;
+
+/**
+ * Invokes a FT method applying semantics based on method annotations. An instance
+ * of this class is created for each method invocation. Some state is shared across
+ * all invocations of a method, including for circuit breakers and bulkheads.
+ */
+public class MethodInvoker implements FtSupplier<Object> {
+    private static final Logger LOGGER = Logger.getLogger(MethodInvoker.class.getName());
+
+    /**
+     * The method being intercepted.
+     */
+    private final Method method;
+
+    /**
+     * Invocation context for the interception.
+     */
+    private final InvocationContext context;
+
+    /**
+     * Helper class to extract information about the method.
+     */
+    private final MethodIntrospector introspector;
+
+    /**
+     * Maps a {@code MethodStateKey} to a {@code MethodState}. The method state returned
+     * caches the FT handler as well as some additional variables. This mapping must
+     * be shared by all instances of this class.
+     */
+    private static final ConcurrentHashMap<MethodStateKey, MethodState> METHOD_STATES = new ConcurrentHashMap<>();
+
+    /**
+     * Start system nanos when handler is called.
+     */
+    private long handlerStartNanos;
+
+    /**
+     * Start system nanos when method {@code proceed()} is called.
+     */
+    private long invocationStartNanos;
+
+    /**
+     * Helidon context in which to run business method.
+     */
+    private final Context helidonContext;
+
+    /**
+     * Jersey's request scope object. Will be non-null if request scope is active.
+     */
+    private RequestScope requestScope;
+
+    /**
+     * Jersey's request scope object.
+     */
+    private RequestContext requestContext;
+
+    /**
+     * CDI's request scope controller used for activation/deactivation.
+     */
+    private RequestContextController requestController;
+
+    /**
+     * Record thread interruption request for later use.
+     */
+    private final AtomicBoolean mayInterruptIfRunning = new AtomicBoolean(false);
+
+    /**
+     * Async thread in used by this invocation. May be {@code null}. We use this
+     * reference for thread interruptions.
+     */
+    private Thread asyncInterruptThread;
+
+    /**
+     * State associated with a method in {@code METHOD_STATES}. This include the
+     * FT handler created for the method.
+     */
+    private static class MethodState {
+        private FtHandlerTyped<Object> handler;
+        private Retry retry;
+        private Bulkhead bulkhead;
+        private CircuitBreaker breaker;
+        private State lastBreakerState;
+        private long breakerTimerOpen;
+        private long breakerTimerClosed;
+        private long breakerTimerHalfOpen;
+        private long startNanos;
+    }
+
+    /**
+     * A key used to lookup {@code MethodState} instances, which include FT handlers.
+     * A class loader is necessary to support multiple applications as seen in the TCKs.
+     * The method class in necessary given that the same method can inherited by different
+     * classes with different FT annotations and should not share handlers. Finally, the
+     * method is main part of the key.
+     */
+    private static class MethodStateKey {
+        private final ClassLoader classLoader;
+        private final Class<?> methodClass;
+        private final Method method;
+
+        MethodStateKey(ClassLoader classLoader, Class<?> methodClass, Method method) {
+            this.classLoader = classLoader;
+            this.methodClass = methodClass;
+            this.method = method;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            MethodStateKey that = (MethodStateKey) o;
+            return classLoader.equals(that.classLoader)
+                    && methodClass.equals(that.methodClass)
+                    && method.equals(that.method);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(classLoader, methodClass, method);
+        }
+    }
+
+    /**
+     * State associated with a method instead of an invocation. Shared by all
+     * invocations of same method.
+     */
+    private final MethodState methodState;
+
+    /**
+     * Future returned by this method invoker. Some special logic to handle async
+     * cancellations and methods returning {@code Future}.
+     *
+     * @param <T> result type of future
+     */
+    @SuppressWarnings("unchecked")
+    class InvokerCompletableFuture<T> extends CompletableFuture<T> {
+
+        /**
+         * If method returns {@code Future}, we let that value pass through
+         * without further processing. See Section 5.2.1 of spec.
+         *
+         * @return value from this future
+         * @throws ExecutionException if this future completed exceptionally
+         * @throws InterruptedException if the current thread was interrupted
+         */
+        @Override
+        public T get() throws InterruptedException, ExecutionException {
+            T value = super.get();
+            if (method.getReturnType() == Future.class) {
+                return ((Future<T>) value).get();
+            }
+            return value;
+        }
+
+        /**
+         * If method returns {@code Future}, we let that value pass through
+         * without further processing. See Section 5.2.1 of spec.
+         *
+         * @param timeout the timeout
+         * @param unit the timeout unit
+         * @return value from this future
+         * @throws CancellationException if this future was cancelled
+         * @throws ExecutionException if this future completed exceptionally
+         * @throws InterruptedException if the current thread was interrupted
+         */
+        @Override
+        public T get(long timeout, TimeUnit unit) throws InterruptedException,
+                ExecutionException, java.util.concurrent.TimeoutException {
+            T value = super.get();
+            if (method.getReturnType() == Future.class) {
+                return ((Future<T>) value).get(timeout, unit);
+            }
+            return value;
+        }
+
+        /**
+         * Overridden to record {@code mayInterruptIfRunning} flag. This flag
+         * is not currently not propagated over a chain of {@code Single<?>}'s.
+         *
+         * @param mayInterruptIfRunning Interrupt flag.
+         * @@return {@code true} if this task is now cancelled.
+         */
+        @Override
+        public boolean cancel(boolean mayInterruptIfRunning) {
+            MethodInvoker.this.mayInterruptIfRunning.set(mayInterruptIfRunning);
+            return super.cancel(mayInterruptIfRunning);
+        }
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param context The invocation context.
+     * @param introspector The method introspector.
+     */
+    public MethodInvoker(InvocationContext context, MethodIntrospector introspector) {
+        this.context = context;
+        this.introspector = introspector;
+        this.method = context.getMethod();
+        this.helidonContext = Contexts.context().orElseGet(Context::create);
+
+        // Create method state using CCL to support multiples apps (like in TCKs)
+        ClassLoader ccl = Thread.currentThread().getContextClassLoader();
+        Objects.requireNonNull(ccl);
+        MethodStateKey methodStateKey = new MethodStateKey(ccl, context.getTarget().getClass(), method);
+        this.methodState = METHOD_STATES.computeIfAbsent(methodStateKey, key -> {
+            MethodState methodState = new MethodState();
+            methodState.lastBreakerState = State.CLOSED;
+            if (introspector.hasCircuitBreaker()) {
+                methodState.breakerTimerOpen = 0L;
+                methodState.breakerTimerClosed = 0L;
+                methodState.breakerTimerHalfOpen = 0L;
+                methodState.startNanos = System.nanoTime();
+            }
+            methodState.handler = createMethodHandler(methodState);
+            return methodState;
+        });
+
+        // Gather information about current request scope if active
+        try {
+            requestController = CDI.current().select(RequestContextController.class).get();
+            requestScope = CDI.current().select(RequestScope.class).get();
+            requestContext = requestScope.current();
+        } catch (Exception e) {
+            requestScope = null;
+            LOGGER.fine(() -> "Request context not active for method " + method
+                    + " on thread " + Thread.currentThread().getName());
+        }
+
+        // Gauges and other metrics for bulkhead and circuit breakers
+        if (isFaultToleranceMetricsEnabled()) {
+            if (introspector.hasCircuitBreaker()) {
+                registerGauge(method, BREAKER_OPEN_TOTAL,
+                        "Amount of time the circuit breaker has spent in open state",
+                        () -> methodState.breakerTimerOpen);
+                registerGauge(method, BREAKER_HALF_OPEN_TOTAL,
+                        "Amount of time the circuit breaker has spent in half-open state",
+                        () -> methodState.breakerTimerHalfOpen);
+                registerGauge(method, BREAKER_CLOSED_TOTAL,
+                        "Amount of time the circuit breaker has spent in closed state",
+                        () -> methodState.breakerTimerClosed);
+            }
+            if (introspector.hasBulkhead()) {
+                registerGauge(method, BULKHEAD_CONCURRENT_EXECUTIONS,
+                        "Number of currently running executions",
+                        () -> methodState.bulkhead.stats().concurrentExecutions());
+                if (introspector.isAsynchronous()) {
+                    registerGauge(method, BULKHEAD_WAITING_QUEUE_POPULATION,
+                            "Number of executions currently waiting in the queue",
+                            () -> methodState.bulkhead.stats().waitingQueueSize());
+                    registerHistogram(
+                            String.format(METRIC_NAME_TEMPLATE,
+                                    method.getDeclaringClass().getName(),
+                                    method.getName(),
+                                    BULKHEAD_WAITING_DURATION),
+                            "Histogram of the time executions spend waiting in the queue.");
+                }
+            }
+        }
+    }
+
+    @Override
+    public String toString() {
+        String s = super.toString();
+        StringBuilder sb = new StringBuilder();
+        sb.append(s.substring(s.lastIndexOf('.') + 1))
+                .append(" ")
+                .append(method.getDeclaringClass().getSimpleName())
+                .append(".")
+                .append(method.getName())
+                .append("()");
+        return sb.toString();
+    }
+
+    /**
+     * Clears {@code METHOD_STATES} map.
+     */
+    static void clearMethodStatesMap() {
+        METHOD_STATES.clear();
+    }
+
+    /**
+     * Invokes a method with one or more FT annotations.
+     *
+     * @return value returned by method.
+     */
+    @Override
+    public Object get() throws Throwable {
+        // Wrap method call with Helidon context
+        Supplier<Single<?>> supplier = () -> {
+            try {
+                return Contexts.runInContextWithThrow(helidonContext,
+                        () -> methodState.handler.invoke(toCompletionStageSupplier(context::proceed)));
+            } catch (Exception e) {
+                return Single.error(e);
+            }
+        };
+
+        // Update metrics before calling method
+        updateMetricsBefore();
+
+        if (introspector.isAsynchronous()) {
+            // Obtain single from supplier
+            Single<?> single = supplier.get();
+
+            // Convert single to CompletableFuture
+            CompletableFuture<?> asyncFuture = single.toStage(true).toCompletableFuture();
+
+            // Create CompletableFuture that is returned to caller
+            CompletableFuture<Object> resultFuture = new InvokerCompletableFuture<>();
+
+            // Update resultFuture based on outcome of asyncFuture
+            asyncFuture.whenComplete((result, throwable) -> {
+                if (throwable != null) {
+                    if (throwable instanceof CancellationException) {
+                        single.cancel();
+                        return;
+                    }
+                    Throwable cause;
+                    if (throwable instanceof ExecutionException) {
+                        cause = map(throwable.getCause());
+                    } else {
+                        cause = map(throwable);
+                    }
+                    updateMetricsAfter(cause);
+                    resultFuture.completeExceptionally(cause);
+                } else {
+                    updateMetricsAfter(null);
+                    resultFuture.complete(result);
+                }
+            });
+
+            // Propagate cancellation of resultFuture to asyncFuture
+            resultFuture.whenComplete((result, throwable) -> {
+                if (throwable instanceof CancellationException) {
+                    asyncFuture.cancel(true);
+                }
+            });
+            return resultFuture;
+        } else {
+            Object result = null;
+            Throwable cause = null;
+            try {
+                // Obtain single from supplier and map to CompletableFuture to handle void methods
+                Single<?> single = supplier.get();
+                CompletableFuture<?> future = single.toStage(true).toCompletableFuture();
+
+                // Synchronously way for result
+                result = future.get();
+            } catch (ExecutionException e) {
+                cause = map(e.getCause());
+            } catch (Throwable t) {
+                cause = map(t);
+            }
+            updateMetricsAfter(cause);
+            if (cause != null) {
+                throw cause;
+            }
+            return result;
+        }
+    }
+
+    /**
+     * Wraps a supplier with additional code to preserve request context (if active)
+     * when running in a different thread. This is required for {@code @Inject} and
+     * {@code @Context} to work properly. Note that it is possible for only CDI's
+     * request scope to be active at this time (e.g. in TCKs).
+     */
+    private FtSupplier<Object> requestContextSupplier(FtSupplier<Object> supplier) {
+        FtSupplier<Object> wrappedSupplier;
+        if (requestScope != null) {                     // Jersey and CDI
+            wrappedSupplier = () -> requestScope.runInScope(requestContext,
+                    (Callable<?>) (() -> {
+                        try {
+                            requestController.activate();
+                            return supplier.get();
+                        } catch (Throwable t) {
+                            throw t instanceof Exception ? ((Exception) t) : new RuntimeException(t);
+                        } finally {
+                            requestController.deactivate();
+                        }
+                    }));
+        } else if (requestController != null) {         // CDI only
+            wrappedSupplier = () -> {
+                try {
+                    requestController.activate();
+                    return supplier.get();
+                } finally {
+                    requestController.deactivate();
+                }
+            };
+        } else {
+            wrappedSupplier = supplier;
+        }
+        return wrappedSupplier;
+    }
+
+    /**
+     * Creates a FT handler for an invocation by inspecting annotations. Handlers
+     * are composed as follows:
+     *
+     *  fallback(retry(circuitbreaker(timeout(bulkhead(method)))))
+     *
+     * Note that timeout includes the time an invocation may be queued in a
+     * bulkhead, so it needs to be before the bulkhead call.
+     *
+     * @param methodState State related to this invocation's method.
+     */
+    private FtHandlerTyped<Object> createMethodHandler(MethodState methodState) {
+        FaultTolerance.TypedBuilder<Object> builder = FaultTolerance.typedBuilder();
+
+        // Create and add bulkhead
+        if (introspector.hasBulkhead()) {
+            methodState.bulkhead = Bulkhead.builder()
+                    .limit(introspector.getBulkhead().value())
+                    .queueLength(introspector.isAsynchronous() ? introspector.getBulkhead().waitingTaskQueue() : 0)
+                    .build();
+            builder.addBulkhead(methodState.bulkhead);
+        }
+
+        // Create and add timeout handler
+        if (introspector.hasTimeout()) {
+            Timeout timeout = Timeout.builder()
+                    .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit()))
+                    .currentThread(!introspector.isAsynchronous())
+                    .build();
+            builder.addTimeout(timeout);
+        }
+
+        // Create and add circuit breaker
+        if (introspector.hasCircuitBreaker()) {
+            methodState.breaker = CircuitBreaker.builder()
+                    .delay(Duration.of(introspector.getCircuitBreaker().delay(),
+                            introspector.getCircuitBreaker().delayUnit()))
+                    .successThreshold(introspector.getCircuitBreaker().successThreshold())
+                    .errorRatio((int) (introspector.getCircuitBreaker().failureRatio() * 100))
+                    .volume(introspector.getCircuitBreaker().requestVolumeThreshold())
+                    .applyOn(mapTypes(introspector.getCircuitBreaker().failOn()))
+                    .skipOn(mapTypes(introspector.getCircuitBreaker().skipOn()))
+                    .build();
+            builder.addBreaker(methodState.breaker);
+        }
+
+        // Create and add retry handler
+        if (introspector.hasRetry()) {
+            Retry retry = Retry.builder()
+                    .retryPolicy(Retry.JitterRetryPolicy.builder()
+                            .calls(introspector.getRetry().maxRetries() + 1)
+                            .delay(Duration.of(introspector.getRetry().delay(),
+                                               introspector.getRetry().delayUnit()))
+                            .jitter(Duration.of(introspector.getRetry().jitter(),
+                                                introspector.getRetry().jitterDelayUnit()))
+                            .build())
+                    .overallTimeout(Duration.of(introspector.getRetry().maxDuration(),
+                                                introspector.getRetry().durationUnit()))
+                    .applyOn(mapTypes(introspector.getRetry().retryOn()))
+                    .skipOn(mapTypes(introspector.getRetry().abortOn()))
+                    .build();
+            builder.addRetry(retry);
+            methodState.retry = retry;      // keep reference to Retry
+        }
+
+        // Create and add fallback handler
+        if (introspector.hasFallback()) {
+            Fallback<Object> fallback = Fallback.builder()
+                    .fallback(throwable -> {
+                        CommandFallback cfb = new CommandFallback(context, introspector, throwable);
+                        return toCompletionStageSupplier(cfb::execute).get();
+                    })
+                    .applyOn(mapTypes(introspector.getFallback().applyOn()))
+                    .skipOn(mapTypes(introspector.getFallback().skipOn()))
+                    .build();
+            builder.addFallback(fallback);
+        }
+
+        return builder.build();
+    }
+
+    /**
+     * Maps an {@link FtSupplier} to a supplier of {@link CompletionStage}.
+     *
+     * @param supplier The supplier.
+     * @return The new supplier.
+     */
+    @SuppressWarnings("unchecked")
+    Supplier<? extends CompletionStage<Object>> toCompletionStageSupplier(FtSupplier<Object> supplier) {
+        return () -> {
+            invocationStartNanos = System.nanoTime();
+
+            CompletableFuture<Object> resultFuture = new CompletableFuture<>();
+            if (introspector.isAsynchronous()) {
+                // Wrap supplier with request context setup
+                FtSupplier wrappedSupplier = requestContextSupplier(supplier);
+
+                // Invoke supplier in new thread and propagate ccl for config
+                ClassLoader ccl = Thread.currentThread().getContextClassLoader();
+                Single<Object> single = Async.create().invoke(() -> {
+                    try {
+                        Thread.currentThread().setContextClassLoader(ccl);
+                        asyncInterruptThread = Thread.currentThread();
+                        return wrappedSupplier.get();
+                    } catch (Throwable t) {
+                        return new InvokerAsyncException(t);        // wraps Throwable
+                    }
+                });
+
+                // Handle async cancellations
+                resultFuture.whenComplete((result, throwable) -> {
+                    if (throwable instanceof CancellationException) {
+                        single.cancel();        // will not interrupt by default
+
+                        // If interrupt was requested, do it manually here
+                        if (mayInterruptIfRunning.get() && asyncInterruptThread != null) {
+                            asyncInterruptThread.interrupt();
+                            asyncInterruptThread = null;
+                        }
+                    }
+                });
+
+                // The result must be Future<?>, {Completable}Future<?> or InvokerAsyncException
+                single.thenAccept(result -> {
+                    try {
+                        // Handle exceptions thrown by an async method
+                        if (result instanceof InvokerAsyncException) {
+                            resultFuture.completeExceptionally(((Exception) result).getCause());
+                        } else if (method.getReturnType() == Future.class) {
+                            // If method returns Future, pass it without further processing
+                            resultFuture.complete(result);
+                        } else if (result instanceof CompletionStage<?>) {     // also CompletableFuture<?>
+                            CompletionStage<Object> cs = (CompletionStage<Object>) result;
+                            cs.whenComplete((o, t) -> {
+                                if (t != null) {
+                                    resultFuture.completeExceptionally(t);
+                                } else {
+                                    resultFuture.complete(o);
+                                }
+                            });
+                        } else {
+                            throw new InternalError("Return type validation failed for method " + method);
+                        }
+                    } catch (Throwable t) {
+                        resultFuture.completeExceptionally(t);
+                    }
+                });
+            } else {
+                try {
+                    resultFuture.complete(supplier.get());
+                    return resultFuture;
+                } catch (Throwable t) {
+                    resultFuture.completeExceptionally(t);
+                }
+            }
+            return resultFuture;
+        };
+    }
+
+    /**
+     * Collects information necessary to update metrics after method is called.
+     */
+    private void updateMetricsBefore() {
+        handlerStartNanos = System.nanoTime();
+
+        if (introspector.hasCircuitBreaker()) {
+            synchronized (method) {
+                // Breaker state may have changed since we recorded it last
+                methodState.lastBreakerState = methodState.breaker.state();
+            }
+        }
+    }
+
+    /**
+     * Update metrics after method is called and depending on outcome.
+     *
+     * @param cause Exception cause or {@code null} if execution successful.
+     */
+    private void updateMetricsAfter(Throwable cause) {
+        if (!isFaultToleranceMetricsEnabled()) {
+            return;
+        }
+
+        synchronized (method) {
+            // Calculate execution time
+            long executionTime = System.nanoTime() - handlerStartNanos;
+
+            // Metrics for retries
+            if (introspector.hasRetry()) {
+                // Have retried the last call?
+                long newValue = methodState.retry.retryCounter();
+                if (updateCounter(method, RETRY_RETRIES_TOTAL, newValue)) {
+                    if (cause == null) {
+                        getCounter(method, RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL).inc();
+                    }
+                } else {
+                    getCounter(method, RETRY_CALLS_SUCCEEDED_NOT_RETRIED_TOTAL).inc();
+                }
+
+                // Update failed calls
+                if (cause != null) {
+                    getCounter(method, RETRY_CALLS_FAILED_TOTAL).inc();
+                }
+            }
+
+            // Timeout
+            if (introspector.hasTimeout()) {
+                getHistogram(method, TIMEOUT_EXECUTION_DURATION).update(executionTime);
+                getCounter(method, cause instanceof TimeoutException
+                        ? TIMEOUT_CALLS_TIMED_OUT_TOTAL
+                        : TIMEOUT_CALLS_NOT_TIMED_OUT_TOTAL).inc();
+            }
+
+            // Circuit breaker
+            if (introspector.hasCircuitBreaker()) {
+                Objects.requireNonNull(methodState.breaker);
+
+                // Update counters based on state changes
+                if (methodState.lastBreakerState == State.OPEN) {
+                    getCounter(method, BREAKER_CALLS_PREVENTED_TOTAL).inc();
+                } else if (methodState.breaker.state() == State.OPEN) {     // closed -> open
+                    getCounter(method, BREAKER_OPENED_TOTAL).inc();
+                }
+
+                // Update succeeded and failed
+                if (cause == null) {
+                    getCounter(method, BREAKER_CALLS_SUCCEEDED_TOTAL).inc();
+                } else if (!(cause instanceof CircuitBreakerOpenException)) {
+                    boolean failure = false;
+                    Class<? extends Throwable>[] failOn = introspector.getCircuitBreaker().failOn();
+                    for (Class<? extends Throwable> c : failOn) {
+                        if (c.isAssignableFrom(cause.getClass())) {
+                            failure = true;
+                            break;
+                        }
+                    }
+
+                    getCounter(method, failure ? BREAKER_CALLS_FAILED_TOTAL
+                            : BREAKER_CALLS_SUCCEEDED_TOTAL).inc();
+                }
+
+                // Update times for gauges
+                switch (methodState.lastBreakerState) {
+                    case OPEN:
+                        methodState.breakerTimerOpen += System.nanoTime() - methodState.startNanos;
+                        break;
+                    case CLOSED:
+                        methodState.breakerTimerClosed += System.nanoTime() - methodState.startNanos;
+                        break;
+                    case HALF_OPEN:
+                        methodState.breakerTimerHalfOpen += System.nanoTime() - methodState.startNanos;
+                        break;
+                    default:
+                        throw new IllegalStateException("Unknown breaker state " + methodState.lastBreakerState);
+                }
+
+                // Update internal state
+                methodState.lastBreakerState = methodState.breaker.state();
+                methodState.startNanos = System.nanoTime();
+            }
+
+            // Bulkhead
+            if (introspector.hasBulkhead()) {
+                Objects.requireNonNull(methodState.bulkhead);
+                Bulkhead.Stats stats = methodState.bulkhead.stats();
+                updateCounter(method, BULKHEAD_CALLS_ACCEPTED_TOTAL, stats.callsAccepted());
+                updateCounter(method, BULKHEAD_CALLS_REJECTED_TOTAL, stats.callsRejected());
+
+                // Update histograms if task accepted
+                if (!(cause instanceof BulkheadException)) {
+                    long waitingTime = invocationStartNanos - handlerStartNanos;
+                    getHistogram(method, BULKHEAD_EXECUTION_DURATION).update(executionTime - waitingTime);
+                    if (introspector.isAsynchronous()) {
+                        getHistogram(method, BULKHEAD_WAITING_DURATION).update(waitingTime);
+                    }
+                }
+            }
+
+            // Global method counters
+            getCounter(method, INVOCATIONS_TOTAL).inc();
+            if (cause != null) {
+                getCounter(method, INVOCATIONS_FAILED_TOTAL).inc();
+            }
+        }
+    }
+
+    /**
+     * Sets the value of a monotonically increasing counter using {@code inc()}.
+     *
+     * @param method The method.
+     * @param name The counter's name.
+     * @param newValue The new value.
+     * @return A value of {@code true} if counter updated, {@code false} otherwise.
+     */
+    private static boolean updateCounter(Method method, String name, long newValue) {
+        Counter counter = getCounter(method, name);
+        long oldValue = counter.getCount();
+        if (newValue > oldValue) {
+            counter.inc(newValue - oldValue);
+            return true;
+        }
+        return false;
+    }
+}

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2020 Oracle and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.helidon.microprofile.faulttolerance;
+
+import java.util.Arrays;
+import java.util.concurrent.ExecutionException;
+
+import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException;
+import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException;
+import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException;
+
+/**
+ * Maps Helidon to MP exceptions.
+ */
+class ThrowableMapper {
+
+    private ThrowableMapper() {
+    }
+
+    /**
+     * Maps a {@code Throwable} in Helidon to its corresponding type in the MP
+     * FT API.
+     *
+     * @param t throwable to map.
+     * @return mapped throwable.
+     */
+    static Throwable map(Throwable t) {
+        if (t instanceof ExecutionException) {
+            t = t.getCause();
+        }
+        if (t instanceof io.helidon.faulttolerance.CircuitBreakerOpenException) {
+            return new CircuitBreakerOpenException(t.getMessage(), t.getCause());
+        }
+        if (t instanceof io.helidon.faulttolerance.BulkheadException) {
+            return new BulkheadException(t.getMessage(), t.getCause());
+        }
+        if (t instanceof java.util.concurrent.TimeoutException) {
+            return new TimeoutException(t.getMessage(), t.getCause());
+        }
+        if (t instanceof java.lang.InterruptedException) {
+            return new TimeoutException(t.getMessage(), t.getCause());
+        }
+        return t;
+    }
+
+    /**
+     * Maps exception types in MP FT to internal ones used by Helidon. Allocates
+     * new array for the purpose of mapping.
+     *
+     * @param types array of {@code Throwable}'s type to map.
+     * @return mapped array.
+     */
+    static Class<? extends Throwable>[] mapTypes(Class<? extends Throwable>[] types) {
+        if (types.length == 0) {
+            return types;
+        }
+        Class<? extends Throwable>[] result = Arrays.copyOf(types, types.length);
+        for (int i = 0; i < types.length; i++) {
+            Class<? extends Throwable> t = types[i];
+            if (t == BulkheadException.class) {
+                result[i] = io.helidon.faulttolerance.BulkheadException.class;
+            } else if (t == CircuitBreakerOpenException.class) {
+                result[i] = io.helidon.faulttolerance.CircuitBreakerOpenException.class;
+            } else if (t == TimeoutException.class) {
+                result[i] = java.util.concurrent.TimeoutException.class;
+            } else {
+                result[i] = t;
+            }
+        }
+        return result;
+    }
+}

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/TimeUtil.java

@@ -1,112 +0,0 @@
-/*
- * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.helidon.microprofile.faulttolerance;
-
-import java.time.temporal.ChronoUnit;
-import java.util.Objects;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Class TimeUtil.
- */
-public class TimeUtil {
-
-    /**
-     * Converts a {@code ChronoUnit} to the equivalent {@code TimeUnit}.
-     *
-     * @param chronoUnit the ChronoUnit to convert
-     * @return the converted equivalent TimeUnit
-     * @throws IllegalArgumentException if {@code chronoUnit} has no equivalent TimeUnit
-     * @throws NullPointerException if {@code chronoUnit} is null
-     */
-    public static TimeUnit chronoUnitToTimeUnit(ChronoUnit chronoUnit) {
-        switch (Objects.requireNonNull(chronoUnit, "chronoUnit")) {
-            case NANOS:
-                return TimeUnit.NANOSECONDS;
-            case MICROS:
-                return TimeUnit.MICROSECONDS;
-            case MILLIS:
-                return TimeUnit.MILLISECONDS;
-            case SECONDS:
-                return TimeUnit.SECONDS;
-            case MINUTES:
-                return TimeUnit.MINUTES;
-            case HOURS:
-                return TimeUnit.HOURS;
-            case DAYS:
-                return TimeUnit.DAYS;
-            default:
-                throw new IllegalArgumentException("No TimeUnit equivalent for ChronoUnit");
-        }
-    }
-
-    /**
-     * Converts this {@code TimeUnit} to the equivalent {@code ChronoUnit}.
-     *
-     * @param timeUnit The TimeUnit
-     * @return the converted equivalent ChronoUnit
-     * @throws IllegalArgumentException if {@code chronoUnit} has no equivalent TimeUnit
-     * @throws NullPointerException if {@code chronoUnit} is null
-     */
-    public static ChronoUnit timeUnitToChronoUnit(TimeUnit timeUnit) {
-        switch (Objects.requireNonNull(timeUnit, "chronoUnit")) {
-            case NANOSECONDS:
-                return ChronoUnit.NANOS;
-            case MICROSECONDS:
-                return ChronoUnit.MICROS;
-            case MILLISECONDS:
-                return ChronoUnit.MILLIS;
-            case SECONDS:
-                return ChronoUnit.SECONDS;
-            case MINUTES:
-                return ChronoUnit.MINUTES;
-            case HOURS:
-                return ChronoUnit.HOURS;
-            case DAYS:
-                return ChronoUnit.DAYS;
-            default:
-                throw new IllegalArgumentException("No ChronoUnit equivalent for TimeUnit");
-        }
-    }
-
-    /**
-     * Converts a duration and its chrono unit to millis.
-     *
-     * @param duration The duration.
-     * @param chronoUnit The unit of the duration.
-     * @return Milliseconds.
-     */
-    public static long convertToMillis(long duration, ChronoUnit chronoUnit) {
-        final TimeUnit timeUnit = chronoUnitToTimeUnit(chronoUnit);
-        return TimeUnit.MILLISECONDS.convert(duration, timeUnit);
-    }
-
-    /**
-     * Converts a duration and its chrono unit to nanos.
-     *
-     * @param duration The duration.
-     * @param chronoUnit The unit of the duration.
-     * @return Nanoseconds.
-     */
-    public static long convertToNanos(long duration, ChronoUnit chronoUnit) {
-        final TimeUnit timeUnit = chronoUnitToTimeUnit(chronoUnit);
-        return TimeUnit.NANOSECONDS.convert(duration, timeUnit);
-    }
-
-    private TimeUtil() {
-    }
-}

microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/TimedHashMap.java

@@ -1,121 +0,0 @@
-/*
- * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.helidon.microprofile.faulttolerance;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
-/**
- * Class TimedHashMap.
- *
- * @param <K> Type of key.
- * @param <V> Type of value.
- */
-public class TimedHashMap<K, V> extends ConcurrentHashMap<K, V> {
-    private static final Logger LOGGER = Logger.getLogger(TimedHashMap.class.getName());
-
-    private static final int THREAD_POOL_SIZE = 3;
-
-    private static final ScheduledExecutorService SCHEDULER =
-        Executors.newScheduledThreadPool(THREAD_POOL_SIZE);
-
-    private final long ttlInMillis;
-
-    private final Map<K, Long> created = new ConcurrentHashMap<>();
-
-    /**
-     * Constructor.
-     *
-     * @param ttlInMillis Time to live in millis for map entries.
-     */
-    public TimedHashMap(long ttlInMillis) {
-        this.ttlInMillis = ttlInMillis;
-        SCHEDULER.scheduleAtFixedRate(this::expireOldEntries, ttlInMillis,
-                                      ttlInMillis, TimeUnit.MILLISECONDS);
-    }
-
-    private void expireOldEntries() {
-        created.keySet()
-               .stream()
-               .filter(k -> System.currentTimeMillis() - created.get(k) > ttlInMillis)
-               .collect(Collectors.toSet())
-               .stream()
-               .forEach(k -> {
-                   LOGGER.fine("Removing expired key " + k);
-                   remove(k);
-                   created.remove(k);
-               });
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        return super.equals(o);
-    }
-
-    @Override
-    public int hashCode() {
-        return super.hashCode();
-    }
-
-    @Override
-    public V put(K key, V value) {
-        created.put(key, System.currentTimeMillis());
-        return super.put(key, value);
-    }
-
-    @Override
-    public void putAll(Map<? extends K, ? extends V> m) {
-        m.keySet()
-         .stream()
-         .forEach(k -> created.put(k, System.currentTimeMillis()));
-        super.putAll(m);
-    }
-
-    @Override
-    public V remove(Object key) {
-        created.remove(key);
-        return super.remove(key);
-    }
-
-    @Override
-    public void clear() {
-        created.clear();
-        super.clear();
-    }
-
-    @Override
-    public V putIfAbsent(K key, V value) {
-        if (!created.containsKey(key)) {
-            created.put(key, System.currentTimeMillis());
-        }
-        return super.putIfAbsent(key, value);
-    }
-
-    @Override
-    public boolean remove(Object key, Object value) {
-        boolean removed = super.remove(key, value);
-        if (removed) {
-            created.remove(key);
-        }
-        return removed;
-    }
-}

microprofile/fault-tolerance/src/main/java/module-info.java

@@ -25,15 +25,12 @@ module io.helidon.microprofile.faulttolerance {
 
     requires io.helidon.common.context;
     requires io.helidon.common.configurable;
+    requires io.helidon.faulttolerance;
     requires io.helidon.microprofile.config;
     requires io.helidon.microprofile.server;
     requires io.helidon.microprofile.metrics;
 
     requires jakarta.enterprise.cdi.api;
-    requires hystrix.core;
-    requires archaius.core;
-    requires commons.configuration;
-    requires failsafe;
 
     requires microprofile.config.api;
     requires microprofile.metrics.api;

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import javax.enterprise.context.Dependent;
 
@@ -32,10 +33,10 @@ import org.eclipse.microprofile.faulttolerance.Fallback;
 @Dependent
 public class AsynchronousBean {
 
-    private boolean called;
+    private AtomicBoolean called = new AtomicBoolean(false);
 
     public boolean wasCalled() {
-        return called;
+        return called.get();
     }
 
     /**
@@ -44,8 +45,8 @@ public class AsynchronousBean {
      * @return A future.
      */
     @Asynchronous
-    public Future<String> async() {
-        called = true;
+    public CompletableFuture<String> async() {
+        called.set(true);
         FaultToleranceTest.printStatus("AsynchronousBean::async", "success");
         return CompletableFuture.completedFuture("success");
     }
@@ -57,10 +58,10 @@ public class AsynchronousBean {
      */
     @Asynchronous
     @Fallback(fallbackMethod = "onFailure")
-    public Future<String> asyncWithFallback() {
-        called = true;
+    public CompletableFuture<String> asyncWithFallback() {
+        called.set(true);
         FaultToleranceTest.printStatus("AsynchronousBean::asyncWithFallback", "failure");
-        throw new RuntimeException("Oops");
+        return CompletableFuture.failedFuture(new RuntimeException("Oops"));
     }
 
     public CompletableFuture<String> onFailure() {
@@ -68,13 +69,32 @@ public class AsynchronousBean {
         return CompletableFuture.completedFuture("fallback");
     }
 
+    /**
+     * Async call with fallback and Future. Fallback should be ignored in this case.
+     *
+     * @return A future.
+     */
+    @Asynchronous
+    @Fallback(fallbackMethod = "onFailureFuture")
+    public Future<String> asyncWithFallbackFuture() {
+        called.set(true);
+        FaultToleranceTest.printStatus("AsynchronousBean::asyncWithFallbackFuture", "failure");
+        return CompletableFuture.failedFuture(new RuntimeException("Oops"));
+    }
+
+    public Future<String> onFailureFuture() {
+        FaultToleranceTest.printStatus("AsynchronousBean::onFailure", "success");
+        return CompletableFuture.completedFuture("fallback");
+    }
+
+
     /**
      * Regular test, not asynchronous.
      *
      * @return A future.
      */
-    public Future<String> notAsync() {
-        called = true;
+    public CompletableFuture<String> notAsync() {
+        called.set(true);
         FaultToleranceTest.printStatus("AsynchronousBean::notAsync", "success");
         return CompletableFuture.completedFuture("success");
     }
@@ -86,7 +106,7 @@ public class AsynchronousBean {
      */
     @Asynchronous
     public CompletionStage<String> asyncCompletionStage() {
-        called = true;
+        called.set(true);
         FaultToleranceTest.printStatus("AsynchronousBean::asyncCompletionStage", "success");
         return CompletableFuture.completedFuture("success");
     }
@@ -99,9 +119,9 @@ public class AsynchronousBean {
     @Asynchronous
     @Fallback(fallbackMethod = "onFailure")
     public CompletionStage<String> asyncCompletionStageWithFallback() {
-        called = true;
+        called.set(true);
         FaultToleranceTest.printStatus("AsynchronousBean::asyncCompletionStageWithFallback", "failure");
-        throw new RuntimeException("Oops");
+        return CompletableFuture.failedFuture(new RuntimeException("Oops"));
     }
 
     /**
@@ -111,7 +131,7 @@ public class AsynchronousBean {
      */
     @Asynchronous
     public CompletableFuture<String> asyncCompletableFuture() {
-        called = true;
+        called.set(true);
         FaultToleranceTest.printStatus("AsynchronousBean::asyncCompletableFuture", "success");
         return CompletableFuture.completedFuture("success");
     }
@@ -124,7 +144,7 @@ public class AsynchronousBean {
     @Asynchronous
     @Fallback(fallbackMethod = "onFailure")
     public CompletableFuture<String> asyncCompletableFutureWithFallback() {
-        called = true;
+        called.set(true);
         FaultToleranceTest.printStatus("AsynchronousBean::asyncCompletableFutureWithFallback", "success");
         return CompletableFuture.completedFuture("success");
     }
@@ -138,7 +158,7 @@ public class AsynchronousBean {
     @Asynchronous
     @Fallback(fallbackMethod = "onFailure")
     public CompletableFuture<String> asyncCompletableFutureWithFallbackFailure() {
-        called = true;
+        called.set(true);
         FaultToleranceTest.printStatus("AsynchronousBean::asyncCompletableFutureWithFallbackFailure", "failure");
         CompletableFuture<String> future = new CompletableFuture<>();
         future.completeExceptionally(new IOException("oops"));

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -34,7 +34,7 @@ public class AsynchronousTest extends FaultToleranceTest {
     public void testAsync() throws Exception {
         AsynchronousBean bean = newBean(AsynchronousBean.class);
         assertThat(bean.wasCalled(), is(false));
-        Future<String> future = bean.async();
+        CompletableFuture<String> future = bean.async();
         future.get();
         assertThat(bean.wasCalled(), is(true));
     }
@@ -43,17 +43,24 @@ public class AsynchronousTest extends FaultToleranceTest {
     public void testAsyncWithFallback() throws Exception {
         AsynchronousBean bean = newBean(AsynchronousBean.class);
         assertThat(bean.wasCalled(), is(false));
-        Future<String> future = bean.asyncWithFallback();
+        CompletableFuture<String> future = bean.asyncWithFallback();
         String value = future.get();
         assertThat(bean.wasCalled(), is(true));
         assertThat(value, is("fallback"));
     }
 
+    @Test
+    public void testAsyncWithFallbackFuture() {
+        AsynchronousBean bean = newBean(AsynchronousBean.class);
+        Future<String> future = bean.asyncWithFallbackFuture();     // fallback ignored with Future
+        assertCompleteExceptionally(future, RuntimeException.class);
+    }
+
     @Test
     public void testAsyncNoGet() throws Exception {
         AsynchronousBean bean = newBean(AsynchronousBean.class);
         assertThat(bean.wasCalled(), is(false));
-        Future<String> future = bean.async();
+        CompletableFuture<String> future = bean.async();
         while (!future.isDone()) {
             Thread.sleep(100);
         }
@@ -64,7 +71,7 @@ public class AsynchronousTest extends FaultToleranceTest {
     public void testNotAsync() throws Exception {
         AsynchronousBean bean = newBean(AsynchronousBean.class);
         assertThat(bean.wasCalled(), is(false));
-        Future<String> future = bean.notAsync();
+        CompletableFuture<String> future = bean.notAsync();
         assertThat(bean.wasCalled(), is(true));
         future.get();
     }

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,7 +17,6 @@
 package io.helidon.microprofile.faulttolerance;
 
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Future;
 
 import javax.enterprise.context.Dependent;
 
@@ -32,67 +31,119 @@ import org.eclipse.microprofile.faulttolerance.Fallback;
 public class BulkheadBean {
 
     static final int CONCURRENT_CALLS = 3;
-
     static final int WAITING_TASK_QUEUE = 3;
+    static final int TOTAL_CALLS = CONCURRENT_CALLS + WAITING_TASK_QUEUE;
 
-    static final int MAX_CONCURRENT_CALLS = CONCURRENT_CALLS + WAITING_TASK_QUEUE;
+    static class ConcurrencyCounter {
+
+        private int currentCalls;
+        private int concurrentCalls;
+        private int totalCalls;
+
+        synchronized void increment() {
+            currentCalls++;
+            if (currentCalls > concurrentCalls) {
+                concurrentCalls = currentCalls;
+            }
+            totalCalls++;
+        }
+
+        synchronized void decrement() {
+            currentCalls--;
+        }
+
+        synchronized int concurrentCalls() {
+            return concurrentCalls;
+        }
+
+        synchronized int totalCalls() {
+            return totalCalls;
+        }
+    }
+
+    private ConcurrencyCounter counter = new ConcurrencyCounter();
+
+    ConcurrencyCounter getCounter() {
+        return counter;
+    }
 
     @Asynchronous
     @Bulkhead(value = CONCURRENT_CALLS, waitingTaskQueue = WAITING_TASK_QUEUE)
-    public Future<String> execute(long sleepMillis) {
-        FaultToleranceTest.printStatus("BulkheadBean::execute", "success");
+    public CompletableFuture<String> execute(long sleepMillis) {
         try {
-            Thread.sleep(sleepMillis);
-        } catch (InterruptedException e) {
-            // falls through
+            counter.increment();
+            FaultToleranceTest.printStatus("BulkheadBean::execute", "success");
+            try {
+                Thread.sleep(sleepMillis);
+            } catch (InterruptedException e) {
+                // falls through
+            }
+            return CompletableFuture.completedFuture(Thread.currentThread().getName());
+        } finally {
+            counter.decrement();
         }
-        return CompletableFuture.completedFuture(Thread.currentThread().getName());
     }
 
     @Asynchronous
     @Bulkhead(value = CONCURRENT_CALLS + 1, waitingTaskQueue = WAITING_TASK_QUEUE + 1)
-    public Future<String> executePlusOne(long sleepMillis) {
-        FaultToleranceTest.printStatus("BulkheadBean::executePlusOne", "success");
+    public CompletableFuture<String> executePlusOne(long sleepMillis) {
         try {
-            Thread.sleep(sleepMillis);
-        } catch (InterruptedException e) {
-            // falls through
+            counter.increment();
+            FaultToleranceTest.printStatus("BulkheadBean::executePlusOne", "success");
+            try {
+                Thread.sleep(sleepMillis);
+            } catch (InterruptedException e) {
+                // falls through
+            }
+            return CompletableFuture.completedFuture(Thread.currentThread().getName());
+        } finally {
+            counter.decrement();
         }
-        return CompletableFuture.completedFuture(Thread.currentThread().getName());
     }
 
     @Asynchronous
     @Bulkhead(value = 2, waitingTaskQueue = 1)
-    public Future<String> executeNoQueue(long sleepMillis) {
-        FaultToleranceTest.printStatus("BulkheadBean::executeNoQueue", "success");
+    public CompletableFuture<String> executeNoQueue(long sleepMillis) {
         try {
-            Thread.sleep(sleepMillis);
-        } catch (InterruptedException e) {
-            // falls through
+            counter.increment();
+            FaultToleranceTest.printStatus("BulkheadBean::executeNoQueue", "success");
+            try {
+                Thread.sleep(sleepMillis);
+            } catch (InterruptedException e) {
+                // falls through
+            }
+            return CompletableFuture.completedFuture(Thread.currentThread().getName());
+        } finally {
+            counter.decrement();
         }
-        return CompletableFuture.completedFuture(Thread.currentThread().getName());
     }
 
+    @Asynchronous
     @Fallback(fallbackMethod = "onFailure")
     @Bulkhead(value = 2, waitingTaskQueue = 1)
-    public Future<String> executeNoQueueWithFallback(long sleepMillis) {
-        FaultToleranceTest.printStatus("BulkheadBean::executeNoQueue", "success");
+    public CompletableFuture<String> executeNoQueueWithFallback(long sleepMillis) {
         try {
-            Thread.sleep(sleepMillis);
-        } catch (InterruptedException e) {
-            // falls through
+            counter.increment();
+            FaultToleranceTest.printStatus("BulkheadBean::executeNoQueue", "success");
+            try {
+                Thread.sleep(sleepMillis);
+            } catch (InterruptedException e) {
+                // falls through
+            }
+            return CompletableFuture.completedFuture(Thread.currentThread().getName());
+        } finally {
+            counter.decrement();
         }
-        return CompletableFuture.completedFuture(Thread.currentThread().getName());
     }
 
-    public String onFailure(long sleepMillis) {
+    public CompletableFuture<String> onFailure(long sleepMillis) {
         FaultToleranceTest.printStatus("BulkheadBean::onFailure()", "success");
-        return Thread.currentThread().getName();
+        return CompletableFuture.completedFuture(Thread.currentThread().getName());
     }
 
     @Asynchronous
     @Bulkhead(value = 1, waitingTaskQueue = 1)
-    public Future<String> executeCancelInQueue(long sleepMillis) {
+    public CompletableFuture<String> executeCancelInQueue(long sleepMillis) {
         FaultToleranceTest.printStatus("BulkheadBean::executeCancelInQueue " + sleepMillis, "success");
         try {
             Thread.sleep(sleepMillis);

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,7 +20,6 @@ import java.util.Arrays;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Future;
 
 import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException;
 import org.junit.jupiter.api.Test;
@@ -40,41 +39,45 @@ public class BulkheadTest extends FaultToleranceTest {
     @Test
     public void testBulkhead() {
         BulkheadBean bean = newBean(BulkheadBean.class);
-        Future<String>[] calls = getAsyncConcurrentCalls(
-            () -> bean.execute(100), BulkheadBean.MAX_CONCURRENT_CALLS);
-        assertThat(getThreadNames(calls).size(), is(BulkheadBean.CONCURRENT_CALLS));
+        CompletableFuture<String>[] calls = getAsyncConcurrentCalls(
+            () -> bean.execute(100), BulkheadBean.TOTAL_CALLS);
+        waitFor(calls);
+        assertThat(bean.getCounter().concurrentCalls(), is(BulkheadBean.CONCURRENT_CALLS));
+        assertThat(bean.getCounter().totalCalls(), is(BulkheadBean.TOTAL_CALLS));
     }
 
     @Test
     public void testBulkheadPlusOne() {
         BulkheadBean bean = newBean(BulkheadBean.class);
-        Future<String>[] calls = getAsyncConcurrentCalls(
-            () -> bean.executePlusOne(100), BulkheadBean.MAX_CONCURRENT_CALLS + 2);
-        assertThat(getThreadNames(calls).size(), is(BulkheadBean.CONCURRENT_CALLS + 1));
+        CompletableFuture<String>[] calls = getAsyncConcurrentCalls(
+            () -> bean.executePlusOne(100), BulkheadBean.TOTAL_CALLS + 2);
+        waitFor(calls);
+        assertThat(bean.getCounter().concurrentCalls(), is(BulkheadBean.CONCURRENT_CALLS + 1));
+        assertThat(bean.getCounter().totalCalls(), is(BulkheadBean.TOTAL_CALLS + 2));
     }
 
     @Test
     public void testBulkheadNoQueue() {
         BulkheadBean bean = newBean(BulkheadBean.class);
-        Future<String>[] calls = getAsyncConcurrentCalls(
+        CompletableFuture<String>[] calls = getAsyncConcurrentCalls(
             () -> bean.executeNoQueue(2000), 10);
-        RuntimeException e = assertThrows(RuntimeException.class, () -> getThreadNames(calls));
+        RuntimeException e = assertThrows(RuntimeException.class, () -> waitFor(calls));
         assertThat(e.getCause().getCause(), instanceOf(BulkheadException.class));
     }
 
     @Test
     public void testBulkheadNoQueueWithFallback() {
         BulkheadBean bean = newBean(BulkheadBean.class);
-        Future<String>[] calls = getAsyncConcurrentCalls(
+        CompletableFuture<String>[] calls = getAsyncConcurrentCalls(
             () -> bean.executeNoQueueWithFallback(2000), 10);
-        getThreadNames(calls);
+        waitFor(calls);
     }
 
     @Test
     public void testBulkheadExecuteCancelInQueue() throws Exception {
         BulkheadBean bean = newBean(BulkheadBean.class);
-        Future<String> f1 = bean.executeCancelInQueue(1000);
-        Future<String> f2 = bean.executeCancelInQueue(2000);    // should never run
+        CompletableFuture<String> f1 = bean.executeCancelInQueue(1000);
+        CompletableFuture<String> f2 = bean.executeCancelInQueue(2000);    // should never run
         boolean b = f2.cancel(true);
         assertTrue(b);
         assertTrue(f2.isCancelled());
@@ -121,8 +124,8 @@ public class BulkheadTest extends FaultToleranceTest {
                 return 0;
             }
         };
-        Future<Integer> f1 = callerBean.submit(callable);
-        Future<Integer> f2 = callerBean.submit(callable);
+        CompletableFuture<Integer> f1 = callerBean.submit(callable);
+        CompletableFuture<Integer> f2 = callerBean.submit(callable);
         assertThat(f1.get() + f2.get(), is(1));
     }
 }

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerBean.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,7 +19,6 @@ package io.helidon.microprofile.faulttolerance;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Future;
 
 import javax.enterprise.context.Dependent;
 
@@ -95,7 +94,7 @@ public class CircuitBreakerBean {
             failureRatio = 1.0,
             delay = 50000,
             failOn = UnitTestException.class)
-    public Future<?> withBulkhead(CountDownLatch started) throws InterruptedException {
+    public CompletableFuture<?> withBulkhead(CountDownLatch started) throws InterruptedException {
         started.countDown();
         FaultToleranceTest.printStatus("CircuitBreakerBean::withBulkhead", "success");
         Thread.sleep(3 * DELAY);

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,10 +16,10 @@
 
 package io.helidon.microprofile.faulttolerance;
 
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
 import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException;
@@ -132,7 +132,7 @@ public class CircuitBreakerTest extends FaultToleranceTest {
         assertFalse(started.await(1000, TimeUnit.MILLISECONDS));
 
         assertThrows(ExecutionException.class, () -> {
-            Future<?> future = bean.withBulkhead(new CountDownLatch(1));
+            CompletableFuture<?> future = bean.withBulkhead(new CountDownLatch(1));
             future.get();
         });
     }

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CommandDataTest.java

@@ -1,58 +0,0 @@
-/*
- * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.helidon.microprofile.faulttolerance;
-
-import java.util.Arrays;
-
-import org.junit.jupiter.api.Test;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
-
-/**
- * Class CommandDataTest.
- */
-public class CommandDataTest {
-
-    @Test
-    public void testSuccessRatio() {
-        CircuitBreakerHelper.CommandData data = new CircuitBreakerHelper.CommandData(6);
-        Arrays.asList(true, true, true, false, false, false).forEach(data::pushResult);
-        assertThat(data.getSuccessRatio(), is(3.0d / 6));
-    }
-
-    @Test
-    public void testFailureRatio() {
-        CircuitBreakerHelper.CommandData data = new CircuitBreakerHelper.CommandData(4);
-        Arrays.asList(true, false, false, false).forEach(data::pushResult);
-        assertThat(data.getFailureRatio(), is(3.0d / 4));
-    }
-
-    @Test
-    public void testPushResult() {
-        CircuitBreakerHelper.CommandData data = new CircuitBreakerHelper.CommandData(2);
-        Arrays.asList(true, false, false, false, true, true).forEach(data::pushResult);     // last two count
-        assertThat(data.getFailureRatio(), is(0.0d));
-    }
-
-    @Test
-    public void testSizeLessCapacity() {
-        CircuitBreakerHelper.CommandData data = new CircuitBreakerHelper.CommandData(6);
-        Arrays.asList(true, false, false).forEach(data::pushResult);
-        assertThat(data.getFailureRatio(), is(-1.0d));    // not enough data
-    }
-}

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java

@@ -16,30 +16,39 @@
 
 package io.helidon.microprofile.faulttolerance;
 
-import java.util.Arrays;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
 import javax.enterprise.inject.literal.NamedLiteral;
 import javax.enterprise.inject.se.SeContainer;
 import javax.enterprise.inject.spi.CDI;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
 
 import io.helidon.microprofile.cdi.HelidonContainer;
 
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 
 /**
  * Class FaultToleranceTest.
  */
 public abstract class FaultToleranceTest {
 
+    private static final long TIMEOUT = 5000;
+    private static final TimeUnit TIMEOUT_UNITS = TimeUnit.MILLISECONDS;
+
     private static SeContainer cdiContainer;
 
     private static final int NUMBER_OF_THREADS = 20;
@@ -58,6 +67,17 @@ public abstract class FaultToleranceTest {
         }
     }
 
+    /**
+     * Clears all internal handlers before running each test. Latest FT spec has
+     * clarified that each method of each class that uses a bulkhead/breaker has
+     * its own state (in application scope). Most of our unit tests assume
+     * independence so we clear this state before running each test.
+     */
+    @BeforeEach
+    public void resetHandlers() {
+        MethodInvoker.clearMethodStatesMap();
+    }
+
     protected static <T> T newBean(Class<T> beanClass) {
         return CDI.current().select(beanClass).get();
     }
@@ -79,17 +99,47 @@ public abstract class FaultToleranceTest {
     }
 
     @SuppressWarnings("unchecked")
-    static <T> Future<T>[] getAsyncConcurrentCalls(Supplier<Future<T>> supplier, int size) {
-        return Stream.generate(() -> supplier.get()).limit(size).toArray(Future[]::new);
+    static <T> CompletableFuture<T>[] getAsyncConcurrentCalls(Supplier<CompletableFuture<T>> supplier, int size) {
+        return Stream.generate(supplier::get).limit(size).toArray(CompletableFuture[]::new);
     }
 
-    static Set<String> getThreadNames(Future<String>[] calls) {
-        return Arrays.asList(calls).stream().map(c -> {
+    static void waitFor(CompletableFuture<String>[] calls) {
+        for (CompletableFuture<String> c : calls) {
             try {
-                return c.get();
+                c.get();
             } catch (Exception e) {
                 throw new RuntimeException(e);
             }
-        }).collect(Collectors.toSet());
+        }
+    }
+
+    static <T> void assertCompleteExceptionally(Future<T> future,
+                                                Class<? extends Throwable> exceptionClass) {
+        assertCompleteExceptionally(future, exceptionClass, null);
+    }
+
+    static <T> void assertCompleteExceptionally(Future<T> future,
+                                                Class<? extends Throwable> exceptionClass,
+                                                String exceptionMessage) {
+        try {
+            future.get(TIMEOUT, TIMEOUT_UNITS);
+            fail("Expected exception: " + exceptionClass.getName());
+        } catch (InterruptedException | TimeoutException e) {
+            fail("Unexpected exception " + e, e);
+        } catch (ExecutionException ee) {
+            assertThat("Cause of ExecutionException", ee.getCause(), instanceOf(exceptionClass));
+            if (exceptionMessage != null) {
+                assertThat(ee.getCause().getMessage(), is(exceptionMessage));
+            }
+        }
+    }
+
+    static void assertCompleteOk(CompletionStage<String> future, String expectedMessage) {
+        try {
+            CompletableFuture<?> cf = future.toCompletableFuture();
+            assertThat(cf.get(TIMEOUT, TIMEOUT_UNITS), is(expectedMessage));
+        } catch (Exception e) {
+            fail("Unexpected exception" + e);
+        }
     }
 }

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsBean.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,6 @@ package io.helidon.microprofile.faulttolerance;
 
 import java.time.temporal.ChronoUnit;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import javax.enterprise.context.Dependent;
@@ -170,7 +169,7 @@ public class MetricsBean {
 
     @Asynchronous
     @Bulkhead(value = 3, waitingTaskQueue = 3)
-    public Future<String> concurrent(long sleepMillis) {
+    public CompletableFuture<String> concurrent(long sleepMillis) {
         FaultToleranceTest.printStatus("MetricsBean::concurrent()", "success");
         try {
             assertThat(getGauge(this,
@@ -185,7 +184,7 @@ public class MetricsBean {
 
     @Asynchronous
     @Bulkhead(value = 3, waitingTaskQueue = 3)
-    public Future<String> concurrentAsync(long sleepMillis) {
+    public CompletableFuture<String> concurrentAsync(long sleepMillis) {
         FaultToleranceTest.printStatus("MetricsBean::concurrentAsync()", "success");
         try {
             assertThat((long) getGauge(this, "concurrentAsync",

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java

@@ -17,7 +17,6 @@
 package io.helidon.microprofile.faulttolerance;
 
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Future;
 
 import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException;
 import org.eclipse.microprofile.metrics.Metadata;
@@ -26,6 +25,7 @@ import org.eclipse.microprofile.metrics.MetricType;
 import org.eclipse.microprofile.metrics.MetricUnits;
 import org.junit.jupiter.api.Test;
 
+import static org.hamcrest.Matchers.greaterThan;
 import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL;
 import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_PREVENTED_TOTAL;
 import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_SUCCEEDED_TOTAL;
@@ -212,7 +212,7 @@ public class MetricsTest extends FaultToleranceTest {
     public void testBreakerTrip() throws Exception {
         MetricsBean bean = newBean(MetricsBean.class);
 
-        for (int i = 0; i < CircuitBreakerBean.REQUEST_VOLUME_THRESHOLD; i++) {
+        for (int i = 0; i < CircuitBreakerBean.REQUEST_VOLUME_THRESHOLD ; i++) {
             assertThrows(RuntimeException.class, () -> bean.exerciseBreaker(false));
         }
         assertThrows(CircuitBreakerOpenException.class, () -> bean.exerciseBreaker(false));
@@ -225,7 +225,7 @@ public class MetricsTest extends FaultToleranceTest {
                    is(0L));
         assertThat(getCounter(bean, "exerciseBreaker",
                                 BREAKER_CALLS_FAILED_TOTAL, boolean.class),
-                   is((long)CircuitBreakerBean.REQUEST_VOLUME_THRESHOLD));
+                   is((long) CircuitBreakerBean.REQUEST_VOLUME_THRESHOLD));
         assertThat(getCounter(bean, "exerciseBreaker",
                                    BREAKER_CALLS_PREVENTED_TOTAL, boolean.class),
                    is(1L));
@@ -335,21 +335,21 @@ public class MetricsTest extends FaultToleranceTest {
     @Test
     public void testBulkheadMetrics() throws Exception {
         MetricsBean bean = newBean(MetricsBean.class);
-        Future<String>[] calls = getAsyncConcurrentCalls(
-            () -> bean.concurrent(100), BulkheadBean.MAX_CONCURRENT_CALLS);
-        getThreadNames(calls);
+        CompletableFuture<String>[] calls = getAsyncConcurrentCalls(
+            () -> bean.concurrent(200), BulkheadBean.TOTAL_CALLS);
+        waitFor(calls);
         assertThat(getGauge(bean, "concurrent",
                               BULKHEAD_CONCURRENT_EXECUTIONS, long.class).getValue(),
                    is(0L));
         assertThat(getCounter(bean, "concurrent",
                                 BULKHEAD_CALLS_ACCEPTED_TOTAL, long.class),
-                   is((long) BulkheadBean.MAX_CONCURRENT_CALLS));
+                   is((long) BulkheadBean.TOTAL_CALLS));
         assertThat(getCounter(bean, "concurrent",
                                 BULKHEAD_CALLS_REJECTED_TOTAL, long.class),
                    is(0L));
         assertThat(getHistogram(bean, "concurrent",
                                   BULKHEAD_EXECUTION_DURATION, long.class).getCount(),
-                   is((long)BulkheadBean.MAX_CONCURRENT_CALLS));
+                   is(greaterThan(0L)));
     }
 
     @Test
@@ -358,14 +358,14 @@ public class MetricsTest extends FaultToleranceTest {
         CompletableFuture<String>[] calls = getConcurrentCalls(
             () -> {
                 try {
-                    return bean.concurrentAsync(100).get();
+                    return bean.concurrentAsync(200).get();
                 } catch (Exception e) {
                     return "failure";
                 }
-            }, BulkheadBean.MAX_CONCURRENT_CALLS);
+            }, BulkheadBean.TOTAL_CALLS);
         CompletableFuture.allOf(calls).get();
         assertThat(getHistogram(bean, "concurrentAsync",
                                   BULKHEAD_EXECUTION_DURATION, long.class).getCount(),
-                   is((long)BulkheadBean.MAX_CONCURRENT_CALLS));
+                   is(greaterThan(0L)));
     }
 }

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,7 +19,6 @@ package io.helidon.microprofile.faulttolerance;
 import java.io.IOException;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
-import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import javax.enterprise.context.Dependent;
@@ -71,13 +70,16 @@ public class RetryBean {
 
     @Asynchronous
     @Retry(maxRetries = 2)
-    public Future<String> retryAsync() {
+    public CompletableFuture<String> retryAsync() {
+        CompletableFuture<String> future = new CompletableFuture<>();
         if (invocations.incrementAndGet() <= 2) {
             printStatus("RetryBean::retryAsync()", "failure");
-            throw new RuntimeException("Oops");
+            future.completeExceptionally(new RuntimeException("Oops"));
+        } else {
+            printStatus("RetryBean::retryAsync()", "success");
+            future.complete("success");
         }
-        printStatus("RetryBean::retryAsync()", "success");
-        return CompletableFuture.completedFuture("success");
+        return future;
     }
 
     @Retry(maxRetries = 4, delay = 100L, jitter = 50L)
@@ -111,13 +113,13 @@ public class RetryBean {
     @Asynchronous
     @Retry(maxRetries = 2)
     public CompletionStage<String> retryWithUltimateSuccess() {
-        if (invocations.incrementAndGet() < 3) {
-        // fails twice
-            throw new RuntimeException("Simulated error");
-        }
-
         CompletableFuture<String> future = new CompletableFuture<>();
-        future.complete("Success");
+        if (invocations.incrementAndGet() < 3) {
+            // fails twice
+            future.completeExceptionally(new RuntimeException("Simulated error"));
+        } else {
+            future.complete("Success");
+        }
         return future;
     }
 }

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,10 +19,6 @@ package io.helidon.microprofile.faulttolerance;
 import java.io.IOException;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import java.util.stream.Stream;
 
 import org.junit.jupiter.params.ParameterizedTest;
@@ -31,19 +27,13 @@ import org.junit.jupiter.params.provider.MethodSource;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
-import static org.junit.jupiter.api.Assertions.fail;
 
 /**
- * Class RetryTest.
+ * Test cases for @Retry.
  */
 public class RetryTest extends FaultToleranceTest {
 
-    // parameterize these for ease of debugging
-    private static final long TIMEOUT = 1000;
-    private static final TimeUnit TIMEOUT_UNITS = TimeUnit.MILLISECONDS;
-
     static Stream<Arguments> createBeans() {
         return Stream.of(
                 Arguments.of(newBean(RetryBean.class), "ManagedRetryBean"),
@@ -70,7 +60,7 @@ public class RetryTest extends FaultToleranceTest {
     @ParameterizedTest(name = "{1}")
     @MethodSource("createBeans")
     public void testRetryAsync(RetryBean bean, String unused) throws Exception {
-        Future<String> future = bean.retryAsync();
+        CompletableFuture<String> future = bean.retryAsync();
         future.get();
         assertThat(bean.getInvocations(), is(3));
     }
@@ -79,7 +69,7 @@ public class RetryTest extends FaultToleranceTest {
     @MethodSource("createBeans")
     public void testRetryWithDelayAndJitter(RetryBean bean, String unused) throws Exception {
         long millis = System.currentTimeMillis();
-        String value = bean.retryWithDelayAndJitter();
+        bean.retryWithDelayAndJitter();
         assertThat(System.currentTimeMillis() - millis, greaterThan(200L));
     }
 
@@ -93,9 +83,8 @@ public class RetryTest extends FaultToleranceTest {
     @ParameterizedTest(name = "{1}")
     @MethodSource("createBeans")
     public void testRetryWithException(RetryBean bean, String unused) throws Exception {
-        final CompletionStage<String> future = bean.retryWithException();
-
-        assertCompleteExceptionally(future, IOException.class, "Simulated error");
+        CompletionStage<String> future = bean.retryWithException();
+        assertCompleteExceptionally(future.toCompletableFuture(), IOException.class, "Simulated error");
         assertThat(bean.getInvocations(), is(3));
     }
 
@@ -105,55 +94,4 @@ public class RetryTest extends FaultToleranceTest {
         assertCompleteOk(bean.retryWithUltimateSuccess(), "Success");
         assertThat(bean.getInvocations(), is(3));
     }
-
-    private void assertCompleteOk(final CompletionStage<String> future, final String expectedMessage) {
-        try {
-            CompletableFuture<?> cf = toCompletableFuture(future);
-            assertThat(cf.get(TIMEOUT, TIMEOUT_UNITS), is(expectedMessage));
-        }
-        catch (Exception e) {
-            fail("Unexpected exception" + e);
-        }
-    }
-
-    private void assertCompleteExceptionally(final CompletionStage<String> future,
-            final Class<? extends Throwable> exceptionClass,
-            final String exceptionMessage) {
-        try {
-            Object result = toCompletableFuture(future).get(TIMEOUT, TIMEOUT_UNITS);
-            fail("Expected exception: " + exceptionClass.getName() + " with message: " + exceptionMessage);
-        }
-        catch (InterruptedException | TimeoutException e) {
-            fail("Unexpected exception " + e, e);
-        }
-        catch (ExecutionException ee) {
-            assertThat("Cause of ExecutionException", ee.getCause(), instanceOf(exceptionClass));
-            assertThat(ee.getCause().getMessage(), is(exceptionMessage));
-        }
-    }
-
-    /**
-     * Returns a future that is completed when the stage is completed and has the same value or exception
-     * as the completed stage. It's supposed to be equivalent to calling
-     * {@link CompletionStage#toCompletableFuture()} but works with any CompletionStage
-     * and doesn't throw {@link java.lang.UnsupportedOperationException}.
-     *
-     * @param <U> The type of the future result
-     * @param stage Stage to convert to a future
-     * @return Future converted from stage
-     */
-    public static <U> CompletableFuture<U> toCompletableFuture(CompletionStage<U> stage) {
-        CompletableFuture<U> future = new CompletableFuture<>();
-        stage.whenComplete((v, e) -> {
-            if (e != null) {
-                future.completeExceptionally(e);
-            }
-            else {
-                future.complete(v);
-            }
-        });
-        return future;
-    }
-
-
 }

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/SchedulerConfigTest.java

@@ -1,58 +0,0 @@
-/*
- * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.helidon.microprofile.faulttolerance;
-
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-
-import io.helidon.common.configurable.ScheduledThreadPoolSupplier;
-import io.helidon.common.context.ContextAwareExecutorService;
-import io.helidon.microprofile.server.Server;
-
-import org.junit.jupiter.api.Test;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.hamcrest.MatcherAssert.assertThat;
-
-/**
- * Testing configuration of {@link CommandScheduler}.
- */
-class SchedulerConfigTest {
-
-    @Test
-    void testNonDefaultConfig() {
-        Server server = null;
-        try {
-            server = Server.builder().port(-1).build();
-            server.start();
-
-            CommandScheduler commandScheduler = CommandScheduler.create(8);
-            assertThat(commandScheduler, notNullValue());
-            ScheduledThreadPoolSupplier poolSupplier = commandScheduler.poolSupplier();
-
-            ScheduledExecutorService service = poolSupplier.get();
-            ContextAwareExecutorService executorService = ((ContextAwareExecutorService) service);
-            ScheduledThreadPoolExecutor stpe = (ScheduledThreadPoolExecutor) executorService.unwrap();
-            assertThat(stpe.getCorePoolSize(), is(8));
-        } finally {
-            if (server != null) {
-                server.stop();
-            }
-        }
-    }
-}

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimedHashMapTest.java

@@ -1,67 +0,0 @@
-/*
- * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.helidon.microprofile.faulttolerance;
-
-import java.util.Map;
-import java.util.stream.IntStream;
-
-import org.junit.jupiter.api.Test;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.is;
-
-/**
- * Class TimedHashMapTest.
- */
-public class TimedHashMapTest extends TimedTest {
-
-    private final long TTL = 500;
-
-    private final Map<String, String> cache = new TimedHashMap<>(TTL);
-
-    @Test
-    public void testExpiration() throws Exception {
-        assertThat(cache.size(), is(0));
-        IntStream.range(0, 10).forEach(
-            i -> cache.put(String.valueOf(i), String.valueOf(i))
-        );
-        assertThat(cache.size(), is(10));
-        Thread.sleep(2 * TTL);
-        assertEventually(() -> assertThat(cache.size(), is(0)));
-    }
-
-    @Test
-    public void testExpirationBatch() throws Exception {
-        assertThat(cache.size(), is(0));
-
-        // First batch
-        IntStream.range(0, 10).forEach(
-            i -> cache.put(String.valueOf(i), String.valueOf(i))
-        );
-        assertThat(cache.size(), is(10));
-        Thread.sleep(TTL / 2);
-
-        // Second batch
-        IntStream.range(10, 20).forEach(
-            i -> cache.put(String.valueOf(i), String.valueOf(i))
-        );
-        assertThat(cache.size(), is(20));
-        Thread.sleep(TTL);
-
-        assertEventually(() -> assertThat(cache.size(), is(0)));
-    }
-}

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,10 +17,12 @@
 package io.helidon.microprofile.faulttolerance;
 
 import java.time.temporal.ChronoUnit;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicLong;
 
 import javax.enterprise.context.Dependent;
 
+import org.eclipse.microprofile.faulttolerance.Asynchronous;
 import org.eclipse.microprofile.faulttolerance.Fallback;
 import org.eclipse.microprofile.faulttolerance.Retry;
 import org.eclipse.microprofile.faulttolerance.Timeout;
@@ -41,6 +43,14 @@ public class TimeoutBean {
         return "failure";
     }
 
+    @Asynchronous
+    @Timeout(value=1000, unit=ChronoUnit.MILLIS)
+    public CompletableFuture<String> forceTimeoutAsync() throws InterruptedException {
+        FaultToleranceTest.printStatus("TimeoutBean::forceTimeoutAsync()", "failure");
+        Thread.sleep(1500);
+        return CompletableFuture.completedFuture("failure");
+    }
+
     @Timeout(value=1000, unit=ChronoUnit.MILLIS)
     public String noTimeout() throws InterruptedException {
         FaultToleranceTest.printStatus("TimeoutBean::noTimeout()", "success");
@@ -48,13 +58,24 @@ public class TimeoutBean {
         return "success";
     }
 
+    @Timeout(value=1000, unit=ChronoUnit.MILLIS)
+    public String forceTimeoutWithCatch() {
+        try {
+            FaultToleranceTest.printStatus("TimeoutBean::forceTimeoutWithCatch()", "failure");
+            Thread.sleep(1500);
+        } catch (InterruptedException e) {
+            // falls through
+        }
+        return null;    // tests special null case
+    }
+
     // See class annotation @Retry(maxRetries = 2)
     @Timeout(value=1000, unit=ChronoUnit.MILLIS)
     public String timeoutWithRetries() throws InterruptedException {
         FaultToleranceTest.printStatus("TimeoutBean::timeoutWithRetries()",
                     duration.get() < 1000 ? "success" : "failure");
         Thread.sleep(duration.getAndAdd(-400));     // needs 2 retries
-        return "success";
+        return duration.get() < 1000 ? "success" : "failure";
     }
 
     @Fallback(fallbackMethod = "onFailure")

microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java

@@ -16,15 +16,17 @@
 
 package io.helidon.microprofile.faulttolerance;
 
+import java.util.concurrent.CompletableFuture;
+
 import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException;
 
 import org.junit.jupiter.api.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.lessThan;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 /**
  * Class TimeoutTest.
@@ -37,12 +39,25 @@ public class TimeoutTest extends FaultToleranceTest {
         assertThrows(TimeoutException.class, bean::forceTimeout);
     }
 
+    @Test
+    public void testForceTimeoutAsync() throws Exception {
+        TimeoutBean bean = newBean(TimeoutBean.class);
+        CompletableFuture<String> future = bean.forceTimeoutAsync();
+        assertCompleteExceptionally(future, TimeoutException.class);
+    }
+
     @Test
     public void testNoTimeout() throws Exception {
         TimeoutBean bean = newBean(TimeoutBean.class);
         assertThat(bean.noTimeout(), is("success"));
     }
 
+    @Test
+    public void testForceTimeoutWithCatch() {
+        TimeoutBean bean = newBean(TimeoutBean.class);
+        assertThrows(TimeoutException.class, bean::forceTimeoutWithCatch);
+    }
+
     @Test
     public void testTimeoutWithRetries() throws Exception {
         TimeoutBean bean = newBean(TimeoutBean.class);
@@ -79,7 +94,7 @@ public class TimeoutTest extends FaultToleranceTest {
         try {
             bean.forceTimeoutLoop();        // cannot interrupt
         } catch (TimeoutException e) {
-            assertThat(System.currentTimeMillis() - start, is(greaterThan(2000L)));
+            assertThat(System.currentTimeMillis() - start, is(greaterThanOrEqualTo(2000L)));
         }
     }
 }

microprofile/tests/arquillian/pom.xml

@@ -66,5 +66,9 @@
             <artifactId>junit</artifactId>
             <version>${version.lib.junit4}</version>
         </dependency>
+        <dependency>
+            <groupId>org.testng</groupId>
+            <artifactId>testng</artifactId>
+        </dependency>
     </dependencies>
 </project>

microprofile/tests/arquillian/src/main/java/io/helidon/microprofile/arquillian/HelidonMethodExecutor.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -30,6 +30,8 @@ import org.jboss.arquillian.test.spi.TestMethodExecutor;
 import org.jboss.arquillian.test.spi.TestResult;
 import org.junit.After;
 import org.junit.Before;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
 
 /**
  * Class HelidonMethodExecutor.
@@ -40,9 +42,14 @@ public class HelidonMethodExecutor implements ContainerMethodExecutor {
     private HelidonCDIInjectionEnricher enricher = new HelidonCDIInjectionEnricher();
 
     /**
-     * Invoke method after enrichment. Inexplicably, the {@code @Before}
-     * and {@code @After} methods are not called when running this
-     * executor. Calling them manually for now.
+     * Invoke method after enrichment.
+     *
+     * - JUnit: Inexplicably, the {@code @Before} and {@code @After} methods are
+     * not called when running this executor, so we call them manually.
+     *
+     * - TestNG: Methods decorated with {@code @BeforeMethod} and {@code AfterMethod}
+     * are called too early, before enrichment takes places. Here we call them
+     * again to make sure instances are initialized properly.
      *
      * @param testMethodExecutor Method executor.
      * @return Test result.
@@ -51,13 +58,13 @@ public class HelidonMethodExecutor implements ContainerMethodExecutor {
         RequestContextController controller = enricher.getRequestContextController();
         try {
             controller.activate();
-            Object object = testMethodExecutor.getInstance();
+            Object instance = testMethodExecutor.getInstance();
             Method method = testMethodExecutor.getMethod();
-            LOGGER.info("Invoking '" + method + "' on " + object);
-            enricher.enrich(object);
-            invokeAnnotated(object, Before.class);
+            LOGGER.info("Invoking '" + method + "' on " + instance);
+            enricher.enrich(instance);
+            invokeBefore(instance);
             testMethodExecutor.invoke(enricher.resolve(method));
-            invokeAnnotated(object, After.class);
+            invokeAfter(instance);
         } catch (Throwable t) {
             return TestResult.failed(t);
         } finally {
@@ -66,15 +73,35 @@ public class HelidonMethodExecutor implements ContainerMethodExecutor {
         return TestResult.passed();
     }
 
+    /**
+     * Invoke before methods.
+     *
+     * @param instance Test instance.
+     */
+    private static void invokeBefore(Object instance) {
+        invokeAnnotated(instance, Before.class);          // Junit
+        invokeAnnotated(instance, BeforeMethod.class);    // TestNG
+    }
+
+    /**
+     * Invoke after methods.
+     *
+     * @param instance Test instance.
+     */
+    private static void invokeAfter(Object instance) {
+        invokeAnnotated(instance, After.class);           // JUnit
+        invokeAnnotated(instance, AfterMethod.class);     // TestNG
+    }
+
     /**
      * Invoke an annotated method.
      *
      * @param object Test instance.
      * @param annotClass Annotation to look for.
      */
-    private void invokeAnnotated(Object object, Class<? extends Annotation> annotClass) {
+    private static void invokeAnnotated(Object object, Class<? extends Annotation> annotClass) {
         Class<?> clazz = object.getClass();
-        Stream.of(clazz.getMethods())
+        Stream.of(clazz.getDeclaredMethods())
                 .filter(m -> m.getAnnotation(annotClass) != null)
                 .forEach(m -> {
                     try {

microprofile/tests/tck/tck-fault-tolerance/src/test/resources/tck-application.yaml

@@ -1,28 +0,0 @@
-#
-# Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# Adjusting some timeouts for Helidon
-#
-org:
-  eclipse:
-    microprofile:
-      fault:
-        tolerance:
-          tck:
-            retry:
-              clientserver:
-                RetryClassLevelClientForMaxRetries/serviceB/Retry/maxDuration: 2000

microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml

@@ -23,7 +23,7 @@
       - system like those in our CI/CD pipeline.
       - No longer commented out - use 'tck-ft' profile to run these tests
       -->
-    <test name="microprofile-fault-tolerance 2.0 TCK">
+    <test name="microprofile-fault-tolerance 2.1.1 TCK">
         <packages>
             <package name="org.eclipse.microprofile.fault.tolerance.tck.*"/>
         </packages>