move bbc tests into bc-tests, remove bc-test

moreover:
- rename TestEngine id from spek2-deprecation to spek2-forgiving

.github/workflows/bbc-ubuntu.yml

@@ -16,6 +16,8 @@ jobs:
           java-version: ${{env.JAVA_VERSION}}
 
       - name: bbcTests
+        env:
+          BC: 1
         run: ./gradlew bbcTests
 
       - name: Upload windows build code coverage

misc/tools/atrium-bc-test/.gitignore

@@ -1 +0,0 @@
-src/old/**

misc/tools/atrium-bc-test/build.gradle

@@ -1,295 +0,0 @@
-// Enable if you need to check the task tree, gr task taskTree --no-repeat
-//plugins {
-//    id "com.dorongold.task-tree" version "1.3.1"
-//}
-
-description = 'Checks that specs from older versions of Atrium can still be run with the components of the current version.'
-
-repositories {
-    maven { url "https://dl.bintray.com/jetbrains/spek" }
-}
-
-configurations {
-    common
-}
-
-dependencies {
-    //spek2
-    common "org.junit.platform:junit-platform-console-standalone:$junit_platform_version"
-    common "org.spekframework.spek2:spek-dsl-jvm:$spek2_version"
-    common "org.spekframework.spek2:spek-runner-junit5:$spek2_version"
-    common "org.spekframework.spek2:spek-runtime-jvm:$spek2_version"
-    common "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
-
-    common mockito(), excludeKotlin
-    common kotlinStdlib()
-
-    implementation configurations.common.dependencies
-    api prefixedProject('core-api-jvm')
-}
-
-def bbcTests = task("bbcTests", group: 'Verification') {
-    description = "runs all binary backward compatibility tests"
-    dependsOn build
-}
-
-def bcTests = task("bcTests", group: 'Verification') {
-    description = "runs all backward compatibility tests which compile against the current code"
-    dependsOn build
-}
-
-['fluent-en_GB', 'infix-en_GB'].each { apiName ->
-    task("bcBbc-$apiName", group: 'Verification') {
-        description = "runs all backward compatibility and binary backward compatibility tests for $apiName"
-        dependsOn build
-    }
-
-    task("bc-$apiName", group: 'Verification') {
-        description = "runs all backward compatibility tests for $apiName"
-        dependsOn build
-    }
-
-    task("bbc-$apiName", group: 'Verification') {
-        description = "runs all binary backward compatibility tests for $apiName"
-        dependsOn build
-    }
-}
-
-def createBcAndBbcTasksForApis(String oldVersion, String forgive, Boolean withBc, Boolean withBbc, String... apis) {
-
-    Task bc_version = null
-    Task bbc_version = null
-    if (withBc) {
-        bc_version = task("bc-$oldVersion", group: 'Verification')
-    }
-    if (withBbc) {
-        bbc_version = task("bbc-$oldVersion", group: 'Verification')
-    }
-
-    apis.each { apiName ->
-        createBcAndBbcTasks(apiName, oldVersion, forgive, withBc, withBbc)
-        def bcBbcApi = project."bcBbc-$apiName"
-        if (withBc) {
-            bc_version.dependsOn "bc-${apiName}-$oldVersion"
-            bcBbcApi.dependsOn "bc-${apiName}-$oldVersion"
-            project."bc-$apiName".dependsOn "bc-${apiName}-$oldVersion"
-        }
-        if (withBbc) {
-            bbc_version.dependsOn "bbc-${apiName}-$oldVersion"
-            bcBbcApi.dependsOn "bbc-${apiName}-$oldVersion"
-            project."bbc-$apiName".dependsOn "bbc-${apiName}-$oldVersion"
-        }
-    }
-}
-
-Task createJacocoReportTask(String apiName, String name, Task runTask) {
-    project.task("jacoco-$name", type: JacocoReport) {
-        sourceSets sourceSets.main
-        executionData runTask
-
-        def projects = apiName.contains('en_GB') ?
-            jacocoMulti.sourceProjects :
-            jacocoMulti.sourceProjects -
-                prefixedProject('translations-en_GB-jvm') -
-                prefixedProject('translations-de_CH-jvm')
-
-        projects.each {
-            sourceSets it.sourceSets.main
-        }
-
-        reports {
-            csv.enabled = false
-            xml.enabled = true
-            xml.destination file("${buildDir}/reports/jacoco/$name/report.xml")
-            html.enabled = true
-            html.destination file("${buildDir}/reports/jacoco/$name/html/")
-        }
-    }
-}
-
-static String getNameUnderline(String apiName, String oldVersion) {
-    return "${apiName.replaceAll('-', '_')}_${oldVersion.replaceAll("\\.", '_')}"
-}
-
-File getSrcDirectory(String apiName, String oldVersion) {
-    return file("${project.projectDir}/src/old/$oldVersion/atrium-api-$apiName")
-}
-
-def createBcAndBbcTasks(String apiName, String oldVersion, String forgive, Boolean withBc, Boolean withBbc) {
-    def nameUnderline = getNameUnderline(apiName, oldVersion)
-    def name = "${apiName}-$oldVersion"
-    def srcName = "src_$nameUnderline"
-    def confCommon = "conf_$nameUnderline"
-    def testJar = "testJar_$nameUnderline"
-    def testJarSources = "testJarSources_$nameUnderline"
-    def srcDirectory = getSrcDirectory(apiName, oldVersion)
-
-    sourceSets {
-        create(srcName) {
-            kotlin.srcDirs = [srcDirectory]
-            resources.srcDirs = [srcDirectory]
-            java.srcDirs = [] // we don't have java sources
-        }
-    }
-
-    configurations {
-        create(confCommon)
-        create(testJar)
-        create(testJarSources)
-    }
-
-    dependencies {
-        def groupId = 'ch.tutteli.atrium'
-        add(testJar, "$groupId:atrium-api-$apiName:$oldVersion:tests") {
-            exclude group: '*'
-        }
-        add(testJarSources, "$groupId:atrium-api-$apiName:$oldVersion:testsources") {
-            exclude group: '*'
-        }
-
-        //TODO tests are now mostly in the common module. However, we cannot unzip this to the jvm project but
-        // require a common module for it. Probably easiest to introduce a second bc-test which is an MPP project
-//            add(testJarSources, "$groupId:atrium-api-$apiName-common:$oldVersion:testsources") {
-//                exclude group: '*'
-//            }
-        add(confCommon, "$groupId:atrium-specs:$oldVersion") {
-            exclude group: 'ch.tutteli.atrium'
-        }
-        //required by atrium-specs
-        add(confCommon, "$groupId:atrium-api-fluent-en_GB:$oldVersion") {
-            exclude group: 'ch.tutteli.atrium'
-        }
-        add(confCommon, prefixedProject("verbs-internal-jvm")) {  //required by specs in the APIs
-            exclude group: '*'
-        }
-
-        //required by atrium-specs
-        add(confCommon, prefixedProject("$apiName-jvm"))
-
-        // dependencies for samples
-        add(confCommon, "org.jetbrains.kotlin:kotlin-test:$kotlin_version")
-        add(confCommon, "org.jetbrains.kotlin:kotlin-test-junit5:$kotlin_version")
-        add(confCommon, "org.junit.jupiter:junit-jupiter-engine:$jupiterVersion")
-
-        add(confCommon, sourceSets.main.runtimeClasspath)
-
-        owner."${srcName}Implementation" configurations[confCommon].dependencies
-    }
-
-    def getArgs = { String scanClassPath ->
-        return [
-            '--scan-class-path', scanClassPath,
-            '--disable-banner',
-            '--fail-if-no-tests',
-            '--include-engine', 'spek2-deprecation',
-            // to execute samples
-            '--include-engine', 'junit-jupiter',
-            '--include-classname', '.*(Samples|Spec)',
-            '--config', forgive,
-            '--details', 'summary'
-        ]
-    }
-
-    apply plugin: 'jacoco'
-
-    jacoco {
-        toolVersion = "$jacoco_tool_version"
-    }
-
-    if (withBbc) {
-        def bbcTask = task("bbc-$name", type: JavaExec, group: 'Verification') {
-            description = "Checks if specs from $apiName $oldVersion can be run against the current version."
-
-            inputs.files configurations[testJar]
-            inputs.files configurations[confCommon]
-
-            classpath configurations[confCommon].asPath
-            classpath configurations[testJar].asPath
-
-            main = 'org.junit.platform.console.ConsoleLauncher'
-            args = getArgs(configurations[testJar].asPath)
-        }
-
-        def jacocoBbcReport = createJacocoReportTask(apiName, "bbc-$name", bbcTask)
-        jacoco {
-            applyTo bbcTask
-        }
-
-        bbcTests.dependsOn jacocoBbcReport
-        bbcTests.dependsOn bbcTask
-    }
-
-    if (withBc) {
-
-        def unzip = task("unzip-$name") {
-            inputs.files configurations[testJarSources]
-
-            outputs.dir srcDirectory
-            doLast {
-                configurations[testJarSources].each {
-                    def jar = it
-                    copy {
-                        from zipTree(jar)
-                        into srcDirectory
-                    }
-                }
-            }
-        }
-
-
-        project."compileSrc_${nameUnderline}Kotlin" {
-            kotlinOptions.suppressWarnings = true
-            dependsOn unzip
-        }
-
-        def bcTask = task("bc-$name", type: JavaExec, group: 'Verification') {
-            description = "Checks if specs from $apiName $oldVersion can be compiled and run against the current version."
-
-            inputs.files sourceSets[srcName].runtimeClasspath
-
-            classpath sourceSets[srcName].runtimeClasspath
-
-            main = 'org.junit.platform.console.ConsoleLauncher'
-            args = getArgs(sourceSets[srcName].output.classesDirs.asPath)
-        }
-
-        def jacocoBcReport = createJacocoReportTask(apiName, "bc-$name", bcTask)
-        jacoco {
-            applyTo bcTask
-        }
-        bcTests.dependsOn jacocoBcReport
-        bcTests.dependsOn bcTask
-    }
-}
-
-def createFixSrcTask(String apiName, String oldVersion, String file, Closure modifyContent) {
-    def nameUnderline070 = getNameUnderline(apiName, oldVersion)
-    def taskSuffix = "$apiName-$oldVersion"
-    def fixSrc = task("fixSrc-$taskSuffix-$file") {
-        doLast {
-            def srcFile = project.file("${getSrcDirectory(apiName, oldVersion)}/ch/tutteli/atrium/api/${apiName.replaceAll('-', '/')}/$file")
-            String newContent = modifyContent(srcFile.getText('UTF-8'))
-            srcFile.write(newContent, 'UTF-8')
-        }
-    }
-    def unzip = project.tasks.getByName("unzip-$taskSuffix")
-    fixSrc.mustRunAfter(unzip)
-    project."compileSrc_${nameUnderline070}Kotlin".dependsOn fixSrc
-}
-
-//@formatter:off
-createBcAndBbcTasksForApis('0.14.0',
-    'forgive=(ch/tutteli/atrium/api/(fluent|infix)/en_GB/(' +
-        '(IterableContainsInOrderOnly.*Spec)|' +
-        '(MapAssertionsSpec.*)' +
-        ').*)',
-    /* withBc = */ true,
-    /* withBbc = */ false,
-    'fluent-en_GB', 'infix-en_GB'
-)
-createBcAndBbcTasksForApis('0.15.0',
-    'forgive=^$',
-    /* withBc = */ true,
-    /* withBbc = */ false,
-    'fluent-en_GB', 'infix-en_GB'
-)

misc/tools/atrium-bc-test/src/main/kotlin/ch/tutteil/atrium/bctest/DeprecationSpek2ExecutionListener.kt

@@ -1,29 +0,0 @@
-package ch.tutteil.atrium.bctest
-
-import org.spekframework.spek2.junit.JUnitEngineExecutionListenerAdapter
-import org.spekframework.spek2.runtime.execution.ExecutionListener
-import org.spekframework.spek2.runtime.execution.ExecutionResult
-import org.spekframework.spek2.runtime.scope.TestScopeImpl
-
-class DeprecationSpek2ExecutionListener(
-    private val listener: JUnitEngineExecutionListenerAdapter,
-    private val forgiveRegex: Regex
-) : ExecutionListener by listener {
-
-    override fun testExecutionFinish(test: TestScopeImpl, result: ExecutionResult) {
-        when (result) {
-            ExecutionResult.Success -> listener.testExecutionFinish(test, ExecutionResult.Success)
-            is ExecutionResult.Failure -> handleFailure(result, test)
-        }
-    }
-
-    private fun handleFailure(result: ExecutionResult.Failure, test: TestScopeImpl) {
-        if (forgiveRegex.matches(test.path.toString())) {
-            println("forgiving ${test.path}")
-            listener.testExecutionFinish(test, ExecutionResult.Success)
-        } else {
-            println("!!!!! path of test in case you want to forgive it failing:\n${test.path}")
-            listener.testExecutionFinish(test, result)
-        }
-    }
-}

misc/tools/atrium-bc-test/src/main/kotlin/ch/tutteil/atrium/bctest/DeprecationSpek2TestEngine.kt

@@ -1,64 +0,0 @@
-package ch.tutteil.atrium.bctest
-
-import org.junit.platform.engine.*
-import org.spekframework.spek2.junit.JUnitEngineExecutionListenerAdapter
-import org.spekframework.spek2.junit.SpekTestDescriptor
-import org.spekframework.spek2.junit.SpekTestDescriptorFactory
-import org.spekframework.spek2.junit.SpekTestEngine
-import org.spekframework.spek2.runtime.SpekRuntime
-import org.spekframework.spek2.runtime.execution.ExecutionRequest
-import java.util.*
-import org.junit.platform.engine.ExecutionRequest as JUnitExecutionRequest
-
-class DeprecationSpek2TestEngine : TestEngine {
-    private val spek = SpekTestEngine()
-    private lateinit var forgiveRegex: Regex
-
-    override fun getId(): String = "spek2-deprecation"
-
-    override fun discover(discoveryRequest: EngineDiscoveryRequest, uniqueId: UniqueId): TestDescriptor {
-        val descriptor = spek.discover(discoveryRequest, uniqueId)
-        val forgive = discoveryRequest.configurationParameters.get("forgive").orElse(null)
-        forgiveRegex = Regex(forgive)
-        require(descriptor.children.isNotEmpty()) {
-            "Could not find any specification, check your runtime classpath"
-        }
-        return descriptor
-    }
-
-    override fun execute(request: JUnitExecutionRequest) {
-        val default = Locale.getDefault()
-        try {
-            Locale.setDefault(Locale.UK)
-
-            runSpekWithCustomListener(request)
-
-        } catch (t: Throwable) {
-            t.printStackTrace()
-            Locale.setDefault(default)
-
-            request.fail(t)
-        }
-    }
-
-    private fun runSpekWithCustomListener(request: JUnitExecutionRequest) {
-        val roots = request.rootTestDescriptor.children
-            .filterIsInstance<SpekTestDescriptor>()
-            .map(SpekTestDescriptor::scope)
-
-        val executionListener = DeprecationSpek2ExecutionListener(
-            JUnitEngineExecutionListenerAdapter(request.engineExecutionListener, SpekTestDescriptorFactory()),
-            forgiveRegex
-        )
-        val executionRequest = ExecutionRequest(roots, executionListener)
-        SpekRuntime().execute(executionRequest)
-    }
-
-    private fun JUnitExecutionRequest.fail(throwable: Throwable) {
-        engineExecutionListener.executionFinished(
-            rootTestDescriptor,
-            TestExecutionResult.failed(throwable)
-        )
-    }
-}
-

misc/tools/atrium-bc-test/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine

@@ -1 +0,0 @@
- ch.tutteil.atrium.bctest.DeprecationSpek2TestEngine

misc/tools/bc-tests/build.gradle.kts

@@ -31,7 +31,7 @@ description =
 
 @Suppress("UNCHECKED_CAST")
 val bcConfigs =
-    (gradle as ExtensionAware).extra.get("bcConfigs") as List<Triple<String, List<Pair<String, List<String>>>, String>>
+    (gradle as ExtensionAware).extra.get("bcConfigs") as List<Triple<String, List<Pair<String, List<String>>>, Pair<Boolean, String>>>
 
 repositories {
     mavenCentral()
@@ -40,6 +40,12 @@ repositories {
 
 val bcTests = tasks.register("bcTests") {
     group = "verification"
+    description = "source backward compatibility tests"
+}
+
+val bbcTests = tasks.register("bbcTests") {
+    group = "verification"
+    description = "binary backward compatibility tests"
 }
 
 subprojects {
@@ -67,8 +73,8 @@ var Project.fixSrc: () -> Unit
         project.extra.set(fixSrcPropertyName, g)
     }
 
-bcConfigs.forEach { (oldVersion, apis, forgivePattern) ->
-
+bcConfigs.forEach { (oldVersion, apis, pair) ->
+    val (includingBbc, forgivePattern) = pair
     fun atrium(module: String): String {
         val artifactNameWithoutPrefix =
             if (module.endsWith("-jvm")) module.substringBeforeLast("-jvm") else module
@@ -220,14 +226,14 @@ bcConfigs.forEach { (oldVersion, apis, forgivePattern) ->
                         inputs.files(compilations["test"].output)
                         inputs.property("forgivePattern", forgivePattern)
 
-                        // declaring it has no outputs, only inputs matter
+                        // declaring it has no outputs, only inputs matter but allow to force run via -PbcForce=1
                         outputs.upToDateWhen {
                             !project.properties.containsKey("bcForce")
                         }
 
                         classpath = compilations["test"].runtimeDependencyFiles
                         main = "org.junit.platform.console.ConsoleLauncher"
-                        args = getBcArgs(compilations["test"].output.classesDirs.asPath, forgivePattern)
+                        args = getConsoleLauncherArgs(compilations["test"].output.classesDirs.asPath, forgivePattern)
                     }
                     bcTests.configure {
                         dependsOn(bcTest)
@@ -236,6 +242,61 @@ bcConfigs.forEach { (oldVersion, apis, forgivePattern) ->
 //                    tasks.named("check").configure {
 //                        dependsOn(bcTest)
 //                    }
+
+                    if (includingBbc) {
+                        val bbcConfName = "bbcConf_${oldVersion}_${apiName}"
+                        val bbcConfNameRuntimeOnly = "bbcConf_${oldVersion}_${apiName}RuntimeOnly"
+
+                        configurations {
+                            create(bbcConfName)
+                            create(bbcConfNameRuntimeOnly)
+                        }
+
+                        dependencies {
+                            add(bbcConfName, atrium("api-$apiName") + ":tests") {
+                                exclude(group = "*")
+                            }
+                            add(bbcConfName, atrium("specs")) {
+                                exclude(group = "ch.tutteli.atrium")
+                            }
+                            add(bbcConfName, project(":bc-tests:test-engine", configuration = "jvmRuntime"))
+                            add(
+                                bbcConfName,
+                                "org.junit.platform:junit-platform-console-standalone:$junitPlatformVersion"
+                            )
+
+                            // required by specs
+                            //might be we have to switch to api as we have defined some of the modules as api in atrium-specs
+                            add(bbcConfName, project(":atrium-fluent-en_GB-jvm"))
+                            add(bbcConfName, project(":atrium-verbs-internal-jvm"))
+
+                            // required by the api tests
+                            add(bbcConfName, project(":atrium-api-$apiName-jvm"))
+                        }
+
+                        // ---------- binary backward compatibility tests ------------
+
+                        val bbcTest = tasks.register<JavaExec>("bbcTest") {
+                            description =
+                                "Checks if specs from $apiName $oldVersion can be run against the current version without recompilation"
+
+                            inputs.files(configurations[bbcConfName])
+                            inputs.property("forgivePattern", forgivePattern)
+
+                            // declaring it has no outputs, only inputs matter but allow to force run via -PbcForce=1
+                            outputs.upToDateWhen {
+                                !project.properties.containsKey("bcForce")
+                            }
+
+                            classpath(configurations[bbcConfName].asPath)
+
+                            main = "org.junit.platform.console.ConsoleLauncher"
+                            args = getConsoleLauncherArgs(configurations[bbcConfName].asPath, forgivePattern)
+                        }
+                        bbcTests.configure {
+                            dependsOn(bbcTest)
+                        }
+                    }
                 }
 
                 sourceSets {
@@ -279,7 +340,6 @@ bcConfigs.forEach { (oldVersion, apis, forgivePattern) ->
 //                    }
                 }
             }
-
             fun memoizeTestFile(testTask: Test) =
                 project.file("${project.buildDir}/test-results/memoize-previous-state-${testTask.name}.txt")
 
@@ -334,14 +394,15 @@ bcConfigs.forEach { (oldVersion, apis, forgivePattern) ->
 
             createUnzipTasks("api-$apiName", "testsources", "Test", targets)
         }
+
     }
 }
 
-fun getBcArgs(scanClassPath: String, forgivePattern: String) = listOf(
+fun getConsoleLauncherArgs(scanClassPath: String, forgivePattern: String) = listOf(
     "--scan-class-path", scanClassPath,
     "--disable-banner",
     "--fail-if-no-tests",
-    "--include-engine", "spek2-deprecation",
+    "--include-engine", "spek2-forgiving",
     "--include-classname", ".*Spec",
     "--config", "forgive=$forgivePattern",
     "--details", "summary"

misc/tools/bc-tests/test-engine/src/jvmMain/kotlin/ch/tutteli/atrium/bctest/Spek2ForgivingTestEngine.kt

@@ -14,7 +14,7 @@ class Spek2ForgivingTestEngine : TestEngine {
     private val spek = SpekTestEngine()
     private lateinit var forgiveRegex: Regex
 
-    override fun getId(): String = "spek2-deprecation"
+    override fun getId(): String = "spek2-forgiving"
 
     override fun discover(discoveryRequest: EngineDiscoveryRequest, uniqueId: UniqueId): TestDescriptor {
         val descriptor = spek.discover(discoveryRequest, uniqueId)

settings.gradle.kts

@@ -8,12 +8,20 @@ buildscript {
 
     val bcConfigs = listOf(
         Triple(
-            "0.14.0", allApisAllTargets, "(ch/tutteli/atrium/api/(fluent|infix)/en_GB/(" +
+            "0.14.0",
+            allApisAllTargets,
+            /* includingBbc= */ false to
+                "(ch/tutteli/atrium/api/(fluent|infix)/en_GB/(" +
                 "(IterableContainsInOrderOnly.*Spec)|" +
                 "(MapAssertionsSpec.*)" +
                 ").*)"
         ),
-        Triple("0.15.0", allApisAllTargets, "^$")
+        Triple(
+            "0.15.0",
+            allApisAllTargets,
+            /* includingBbc= */  false to
+                "^$"
+        )
     )
     (gradle as ExtensionAware).extra.apply {
         apply {
@@ -31,7 +39,7 @@ if (System.getenv("BC") != null) {
 
     @Suppress("UNCHECKED_CAST")
     val bcConfigs =
-        (gradle as ExtensionAware).extra.get("bcConfigs") as List<Triple<String, List<Pair<String, List<String>>>, String>>
+        (gradle as ExtensionAware).extra.get("bcConfigs") as List<Triple<String, List<Pair<String, List<String>>>, Pair<Boolean, String>>>
     bcConfigs.forEach { (oldVersion, apis, _) ->
         includeBc(oldVersion, "specs")
         apis.forEach { (apiName, _) ->