Junit5中如何控制测试执行结果？

Question

我在 JUnit5 中创建了一个新注释，它创建了相同测试的复制流，并根据某些条件运行或禁用它们。

但是，如果至少有一次迭代失败，它会自动使整个测试套件失败，我希望能够控制父测试执行结果。

例如，我想设置如果通过了一定数量的副本，那么整个套件应该通过。 有没有办法做到这一点？

这是我的代码：

public class Test {

private static int i = 0;

    @FlakyTest(maxIterations = 10, maxFailuresRate = 0.4)
    public void test() {
        if(i++ == 0){
            assert false;
        } else {
            assert true;
        }
    }
}


import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@TestTemplate
@ExtendWith(FlakyTestRunner.class)
public @interface FlakyTest {

    int maxIterations() default 6;
    double maxFailuresRate() default 0.2;
}


import org.junit.jupiter.api.extension.AfterTestExecutionCallback;
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
import org.junit.jupiter.api.extension.ExecutionCondition;
import org.junit.jupiter.api.extension.Extension;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import static flaky.FlakyTestRunner.didPassedFailureRate;
import static org.junit.platform.commons.util.AnnotationUtils.isAnnotated;

public class FlakyTestRunner implements TestTemplateInvocationContextProvider, AfterTestExecutionCallback {

    public static int iteration = 0;
    public static int maxIterations;
    public static double maxFailuresRate;
    private static Map<Integer, Boolean> iterationsResultsMap = new HashMap<>();

    @Override
    public boolean supportsTestTemplate(ExtensionContext extensionContext) {
        return isAnnotated(extensionContext.getTestMethod(), FlakyTest.class);
    }

    @Override
    public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContexts(ExtensionContext extensionContext) {
        maxIterations = extensionContext.getElement().get().getAnnotation(FlakyTest.class).maxIterations();
        maxFailuresRate = extensionContext.getElement().get().getAnnotation(FlakyTest.class).maxFailuresRate();
        List invocationContexts = new ArrayList<TestTemplateInvocationContext>();
        for (int i = 0; i < maxIterations; i++) {
            invocationContexts.add(new FlakyIterationRunnerTemplateInvocationContext());
        }
        return invocationContexts.stream();
    }

    @Override
    public void afterTestExecution(ExtensionContext extensionContext) {
        iterationsResultsMap.put(iteration, !extensionContext.getExecutionException().isPresent());
    }

    public static boolean didPassedFailureRate() {
        if (iteration > 2) {
            return getFailedTestsRate() >= maxFailuresRate;
        }
        return false;
    }

    private static double getFailedTestsRate() {
        int sum = iterationsResultsMap.values()
                                      .stream()
                                      .mapToInt(successFlag -> successFlag ? 0 : 1)
                                      .sum();
        return ((double) sum) / maxIterations;
    }
}

class FlakyIterationRunnerTemplateInvocationContext implements TestTemplateInvocationContext {

    @Override
    public List<Extension> getAdditionalExtensions() {
        List<Extension> extensions = new ArrayList<>();
        extensions.add(new FlakyIterationRunnerExecutionCondition());
        return extensions;
    }
}

class FlakyIterationRunnerExecutionCondition implements ExecutionCondition {

    @Override
    public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext extensionContext) {
        FlakyTestRunner.iteration++;
        if (FlakyTestRunner.iteration <= FlakyTestRunner.maxIterations && !didPassedFailureRate()) {
            return ConditionEvaluationResult.enabled("Passed");
        }
        return ConditionEvaluationResult.disabled("Iteration number: " + FlakyTestRunner.iteration + ", did passed failure rate? " + didPassedFailureRate()
                + ". Max failures rate allowed - " + FlakyTestRunner.maxFailuresRate);
    }
}

Answer 1

我通常认为“flakey 测试”是一个坏主意，并且几乎总是可以进行某种程度的重构来消除对此类事情的需求。但是，如果这是您想要采用的方法，我认为您可以通过让您的扩展实现 TestExecutionExceptionHandler 来实现它，以便您可以决定 AssertionError 何时应该导致测试失败。我不知道这是否是一个特别好的实现，但我想这会做你想要的：

public class FlakyTestRunner implements TestTemplateInvocationContextProvider, TestExecutionExceptionHandler  {

    private Map<Method,MultiIterationResult> results = new HashMap<>();

    @Override
    public boolean supportsTestTemplate(ExtensionContext extensionContext) {
        return isAnnotated(extensionContext.getTestMethod(), FlakyTest.class);
    }

    @Override
    public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContexts(
        ExtensionContext extensionContext) {
        int maxIterations = extensionContext.getElement().get().getAnnotation(FlakyTest.class).maxIterations();
        double maxFailuresRate = extensionContext.getElement().get().getAnnotation(FlakyTest.class).maxFailuresRate();
        results.put(extensionContext.getTestMethod().get(), new MultiIterationResult(maxIterations, maxFailuresRate));
        List invocationContexts = new ArrayList<TestTemplateInvocationContext>();
        for (int i = 0; i < maxIterations; i++) {
            invocationContexts.add(new FlakyIterationRunnerTemplateInvocationContext());
        }
        return invocationContexts.stream();
    }

    @Override
    public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable {
        results.get(context.getTestMethod().orElseThrow(() -> throwable)).handleFailure(throwable);
    }

    private class MultiIterationResult {
        private final int iterations;
        private final double failureThreshold;
        private int failCount = 0;

        public MultiIterationResult(int iterations, double failureThreshold) {
            this.iterations = iterations;
            this.failureThreshold = failureThreshold;
        }

        public void handleFailure(Throwable throwable) throws Throwable {
            failCount++;
            if((double)failCount/iterations > failureThreshold) {
                throw throwable;
            }
        }
    }

    private  class FlakyIterationRunnerTemplateInvocationContext implements TestTemplateInvocationContext {
    }
}