New test case architecture

sequoia-wot-vectors/src/test/kotlin/org/sequoia_pgp/wot/suite/BestViaRoot.kt

@@ -6,7 +6,7 @@ package org.sequoia_pgp.wot.suite
 
 import org.sequoia_pgp.wot.vectors.BestViaRootVectors
 
-class BestViaRoot : TestCase(BestViaRootVectors()) {
+class BestViaRoot : SimpleTestCase(BestViaRootVectors()) {
 
     override fun arguments(): Array<String> {
         val v = vectors as BestViaRootVectors

sequoia-wot-vectors/src/test/kotlin/org/sequoia_pgp/wot/suite/ExecutionCallback.kt

@@ -0,0 +1,7 @@
+package org.sequoia_pgp.wot.suite
+
+import org.sequoia_pgp.wot.vectors.ArtifactVectors
+
+interface ExecutionCallback {
+    fun execute(vectors: ArtifactVectors, arguments: Array<String>): Pair<String, Int>
+}

sequoia-wot-vectors/src/test/kotlin/org/sequoia_pgp/wot/suite/SimpleTestCase.kt

@@ -0,0 +1,25 @@
+// SPDX-FileCopyrightText: 2023 Paul Schaub <vanitasvitae@fsfe.org>
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+package org.sequoia_pgp.wot.suite
+
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.MethodSource
+import org.sequoia_pgp.wot.vectors.ArtifactVectors
+
+/**
+ * Simple test case, which tests a single WOT query invocation and compares the output to the expected result.
+ */
+abstract class SimpleTestCase(vectors: ArtifactVectors): TestCase(vectors)  {
+
+    @ParameterizedTest
+    @MethodSource("instances")
+    fun execute(callback: ExecutionCallback) {
+        assertResultEquals(callback, arguments(), expectedOutput().first, expectedOutput().second)
+    }
+
+    abstract fun arguments(): Array<String>
+
+    abstract fun expectedOutput(): Pair<String, Int>
+}

sequoia-wot-vectors/src/test/kotlin/org/sequoia_pgp/wot/suite/TestCase.kt

@@ -1,43 +1,44 @@
-// SPDX-FileCopyrightText: 2023 Paul Schaub <vanitasvitae@fsfe.org>
-//
-// SPDX-License-Identifier: BSD-3-Clause
-
 package org.sequoia_pgp.wot.suite
 
 import org.junit.jupiter.api.Named
-import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.Arguments
-import org.junit.jupiter.params.provider.MethodSource
 import org.sequoia_pgp.wot.suite.harness.ExecutableHarness
 import org.sequoia_pgp.wot.suite.harness.WotCLIHarness
 import org.sequoia_pgp.wot.vectors.ArtifactVectors
 import java.io.File
 import kotlin.test.assertEquals
 
-abstract class TestCase(val vectors: ArtifactVectors) {
+/**
-
+ * Test case which allows to query the given [ArtifactVectors] using different WOT implementations.
-    interface ExecutionCallback {
+ *
-        fun execute(vectors: ArtifactVectors, arguments: Array<String>): Pair<String, Int>
+ * To implement a concrete test case, extend this class and add one or more methods with the following signature:
-    }
+ * ```
-
+ * @ParameterizedTest
-    @ParameterizedTest
+ * @MethodSource("instances")
-    @MethodSource("instances")
+ * fun exampleTest(callback: ExecutionCallback) {
-    fun execute(callback: ExecutionCallback) {
+ *     val arguments = arrayOf("--keyring", keyRingPath(), "--full", "identify", ...)
-        val arguments = arguments()
+ *     val expectedOutput = "[✓] 2AB08C06FC795AC26673B23CAD561ABDCBEBFDF0 <target@example.org>: fully authenticated (100%)" +
-        val expectedOutput = expectedOutput()
+ *             "..."
-
+ *     assertResultEquals(callback, arguments, expectedOutput, 0)
-        val result = callback.execute(vectors, arguments)
+ * }
-        assertEquals(expectedOutput.first, result.first)
+ * ```
-        assertEquals(expectedOutput.second, result.second)
+ */
-    }
+open class TestCase(val vectors: ArtifactVectors) {
-
-    abstract fun arguments(): Array<String>
-
-    abstract fun expectedOutput(): Pair<String, Int>
 
     internal fun keyRingPath(): String =
             vectors.tempKeyRingFile.absolutePath
 
+    fun assertResultEquals(
+            callback: ExecutionCallback,
+            arguments: Array<String>,
+            expectedOutput: String,
+            expectedExitCode: Int) {
+        val result = callback.execute(vectors, arguments)
+
+        assertEquals(expectedOutput, result.first)
+        assertEquals(expectedExitCode, result.second)
+    }
+
     companion object {
         @JvmStatic
         fun instances(): List<Arguments> {

sequoia-wot-vectors/src/test/kotlin/org/sequoia_pgp/wot/suite/harness/ExecutableHarness.kt

@@ -5,7 +5,7 @@
 package org.sequoia_pgp.wot.suite.harness
 
 import org.bouncycastle.util.io.Streams
-import org.sequoia_pgp.wot.suite.TestCase
+import org.sequoia_pgp.wot.suite.ExecutionCallback
 import org.sequoia_pgp.wot.vectors.ArtifactVectors
 import java.io.ByteArrayOutputStream
 
@@ -16,8 +16,8 @@ import java.io.ByteArrayOutputStream
  */
 class ExecutableHarness(val executable: String, val environment: Array<String>): Harness() {
 
-    override fun runner(): TestCase.ExecutionCallback {
+    override fun runner(): ExecutionCallback {
-        return object: TestCase.ExecutionCallback {
+        return object: ExecutionCallback {
 
             override fun execute(vectors: ArtifactVectors, arguments: Array<String>): Pair<String, Int> {
                 val command = arrayOf(executable).plus(arguments)

sequoia-wot-vectors/src/test/kotlin/org/sequoia_pgp/wot/suite/harness/Harness.kt

@@ -4,15 +4,16 @@
 
 package org.sequoia_pgp.wot.suite.harness
 
-import org.sequoia_pgp.wot.suite.TestCase
+import org.sequoia_pgp.wot.suite.ExecutionCallback
+import org.sequoia_pgp.wot.suite.SimpleTestCase
 
 /**
- * Abstract class to produce [TestCase.ExecutionCallback] instances for WOT CLI implementations.
+ * Abstract class to produce [SimpleTestCase.ExecutionCallback] instances for WOT CLI implementations.
  */
 abstract class Harness {
 
     /**
-     * Return a [TestCase.ExecutionCallback] which executes a [TestCase] using a custom WOT implementation.
+     * Return a [SimpleTestCase.ExecutionCallback] which executes a [SimpleTestCase] using a custom WOT implementation.
      */
-    abstract fun runner(): TestCase.ExecutionCallback
+    abstract fun runner(): ExecutionCallback
 }

sequoia-wot-vectors/src/test/kotlin/org/sequoia_pgp/wot/suite/harness/WotCLIHarness.kt

@@ -5,7 +5,7 @@
 package org.sequoia_pgp.wot.suite.harness
 
 import org.pgpainless.wot.cli.WotCLI
-import org.sequoia_pgp.wot.suite.TestCase
+import org.sequoia_pgp.wot.suite.ExecutionCallback
 import org.sequoia_pgp.wot.vectors.ArtifactVectors
 import java.io.ByteArrayOutputStream
 import java.io.PrintStream
@@ -15,8 +15,8 @@ import java.io.PrintStream
  */
 class WotCLIHarness: Harness() {
 
-    override fun runner(): TestCase.ExecutionCallback {
+    override fun runner(): ExecutionCallback {
-        return object: TestCase.ExecutionCallback {
+        return object: ExecutionCallback {
 
             override fun execute(vectors: ArtifactVectors, arguments: Array<String>): Pair<String, Int> {
                 val origStdout = System.out