[sinttest] Add tagging of tests with references to (XMPP) specifications

A new annotation is introduced (`SpecificationReference`) that can be used to annotate a SINT test class

The properties are available in the annotation:
- `document`: Identifier for a specification document, such as 'RFC 6120' or 'XEP-0485'

The pre-existing `SmackIntegrationTest` annotation has now received two new properties:
- `section`: Identifier for a section (or paragraph), such as '6.2.1'
- `quote`: A quotation of relevant text from the section
These are expected to be used in context of the `SpecificationReference` annotation.

The SINT execution framework is modified so that two new configuration options are available:
- `enabledSpecifications`
- `disabledSpecifications`

These operate on the value of the `document` property of the annotation. Their usage is comparable
to that of the pre-existing `enabledTests` and `disabledTest` configuration options.

Execution output now includes the document, section and quote that's on the annotated test, when
the test fails. This allows an end-user to easily correspond a test failure with a particular
specification.

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/Configuration.java

@@ -111,6 +111,10 @@ public final class Configuration {
 
     private final Map<String, Set<String>> disabledTestsMap;
 
+    public final Set<String> enabledSpecifications;
+
+    public final Set<String> disabledSpecifications;
+
     public final String defaultConnectionNickname;
 
     public final Set<String> enabledConnections;
@@ -181,6 +185,8 @@ public final class Configuration {
         this.enabledTestsMap = convertTestsToMap(enabledTests);
         this.disabledTests =  CollectionUtil.nullSafeUnmodifiableSet(builder.disabledTests);
         this.disabledTestsMap = convertTestsToMap(disabledTests);
+        this.enabledSpecifications = CollectionUtil.nullSafeUnmodifiableSet(builder.enabledSpecifications);
+        this.disabledSpecifications = CollectionUtil.nullSafeUnmodifiableSet(builder.disabledSpecifications);
         this.defaultConnectionNickname = builder.defaultConnectionNickname;
         this.enabledConnections = builder.enabledConnections;
         this.disabledConnections = builder.disabledConnections;
@@ -252,6 +258,10 @@ public final class Configuration {
 
         private Set<String> disabledTests;
 
+        private Set<String> enabledSpecifications;
+
+        private Set<String> disabledSpecifications;
+
         private String defaultConnectionNickname;
 
         private Set<String> enabledConnections;
@@ -378,6 +388,16 @@ public final class Configuration {
             return this;
         }
 
+        public Builder setEnabledSpecifications(String enabledSpecificationsString) {
+            enabledSpecifications = getSpecificationSetFrom(enabledSpecificationsString);
+            return this;
+        }
+
+        public Builder setDisabledSpecifications(String disabledSpecificationsString) {
+            disabledSpecifications = getSpecificationSetFrom(disabledSpecificationsString);
+            return this;
+        }
+
         public Builder setDefaultConnection(String defaultConnectionNickname) {
             this.defaultConnectionNickname = defaultConnectionNickname;
             return this;
@@ -523,6 +543,8 @@ public final class Configuration {
         builder.setDebugger(properties.getProperty("debugger"));
         builder.setEnabledTests(properties.getProperty("enabledTests"));
         builder.setDisabledTests(properties.getProperty("disabledTests"));
+        builder.setEnabledSpecifications(properties.getProperty("enabledSpecifications"));
+        builder.setDisabledSpecifications(properties.getProperty("disabledSpecifications"));
         builder.setDefaultConnection(properties.getProperty("defaultConnection"));
         builder.setEnabledConnections(properties.getProperty("enabledConnections"));
         builder.setDisabledConnections(properties.getProperty("disabledConnections"));
@@ -587,6 +609,10 @@ public final class Configuration {
         });
     }
 
+    private static Set<String> getSpecificationSetFrom(String input) {
+        return split(input, Configuration::normalizeSpecification);
+    }
+
     private static Map<String, Set<String>> convertTestsToMap(Set<String> tests) {
         Map<String, Set<String>> res = new HashMap<>();
         for (String test : tests) {
@@ -695,4 +721,34 @@ public final class Configuration {
         return contains(method, disabledTestsMap);
     }
 
+    public boolean isSpecificationEnabled(String specification) {
+        if (enabledSpecifications.isEmpty()) {
+            return true;
+        }
+
+        if (specification == null) {
+            return false;
+        }
+
+        return enabledSpecifications.contains(normalizeSpecification(specification));
+    }
+
+    public boolean isSpecificationDisabled(String specification) {
+        if (disabledSpecifications.isEmpty()) {
+            return false;
+        }
+
+        if (specification == null) {
+            return false;
+        }
+
+        return disabledSpecifications.contains(normalizeSpecification(specification));
+    }
+
+    static String normalizeSpecification(String specification) {
+        if (specification == null || specification.isBlank()) {
+            return null;
+        }
+        return specification.replaceAll("\\s", "").toUpperCase();
+    }
 }

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/SmackIntegrationTestFramework.java

@@ -44,6 +44,8 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -71,6 +73,7 @@ import org.igniterealtime.smack.inttest.Configuration.AccountRegistration;
 import org.igniterealtime.smack.inttest.annotations.AfterClass;
 import org.igniterealtime.smack.inttest.annotations.BeforeClass;
 import org.igniterealtime.smack.inttest.annotations.SmackIntegrationTest;
+import org.igniterealtime.smack.inttest.annotations.SpecificationReference;
 import org.reflections.Reflections;
 import org.reflections.scanners.MethodAnnotationsScanner;
 import org.reflections.scanners.MethodParameterScanner;
@@ -128,10 +131,21 @@ public class SmackIntegrationTestFramework {
         final int exitStatus;
         if (failedTests > 0) {
             LOGGER.warning("💀 The following " + failedTests + " tests failed! 💀");
+            final SortedSet<String> bySpecification = new TreeSet<>();
             for (FailedTest failedTest : testRunResult.failedIntegrationTests) {
                 final Throwable cause = failedTest.failureReason;
                 LOGGER.log(Level.SEVERE, failedTest.concreteTest + " failed: " + cause, cause);
+                if (failedTest.concreteTest.method.isAnnotationPresent(SpecificationReference.class)) {
+                    final String specificationReference = getSpecificationReference(failedTest.concreteTest.method);
+                    if (specificationReference != null) {
+                        bySpecification.add("- " + specificationReference + " (as tested by '" + failedTest.concreteTest + "')");
                     }
+                }
+            }
+            if (!bySpecification.isEmpty()) {
+                LOGGER.log(Level.SEVERE, "The failed tests correspond to the following specifications:" + System.lineSeparator() + String.join(System.lineSeparator(), bySpecification));
+            }
+
             exitStatus = 2;
         } else {
             LOGGER.info("All possible Smack Integration Tests completed successfully. \\o/");
@@ -149,6 +163,24 @@ public class SmackIntegrationTestFramework {
         System.exit(exitStatus);
     }
 
+    private static String getSpecificationReference(Method method) {
+        final SpecificationReference spec = method.getDeclaringClass().getAnnotation(SpecificationReference.class);
+        if (spec == null || spec.document().isBlank()) {
+            return null;
+        }
+        String line = spec.document().trim();
+
+        final SmackIntegrationTest test = method.getAnnotation(SmackIntegrationTest.class);
+        if (!test.section().isBlank()) {
+            line += " section " + test.section().trim();
+        }
+        if (!test.quote().isBlank()) {
+            line += ":\t\"" + test.quote().trim() + "\"";
+        }
+        assert !line.isBlank();
+        return line;
+    }
+
     public SmackIntegrationTestFramework(Configuration configuration) {
         this.config = configuration;
     }
@@ -297,6 +329,26 @@ public class SmackIntegrationTestFramework {
                 continue;
             }
 
+            final String specification;
+            if (testClass.isAnnotationPresent(SpecificationReference.class)) {
+                final SpecificationReference specificationReferenceAnnotation = testClass.getAnnotation(SpecificationReference.class);
+                specification = Configuration.normalizeSpecification(specificationReferenceAnnotation.document());
+            } else {
+                specification = null;
+            }
+
+            if (!config.isSpecificationEnabled(specification)) {
+                DisabledTestClass disabledTestClass = new DisabledTestClass(testClass, "Skipping test method " + testClass + " because it tests a specification ('" + specification + "') that is not enabled");
+                testRunResult.disabledTestClasses.add(disabledTestClass);
+                continue;
+            }
+
+            if (config.isSpecificationDisabled(specification)) {
+                DisabledTestClass disabledTestClass = new DisabledTestClass(testClass, "Skipping test method " + testClass + " because it tests a specification ('" + specification + "') that is disabled");
+                testRunResult.disabledTestClasses.add(disabledTestClass);
+                continue;
+            }
+
             final Constructor<? extends AbstractSmackIntTest> cons;
             try {
                 cons = testClass.getConstructor(SmackIntegrationTestEnvironment.class);

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SmackIntegrationTest.java

@@ -31,4 +31,18 @@ public @interface SmackIntegrationTest {
 
     int connectionCount() default -1;
 
+    /**
+     * Unique identifier for a section (or paragraph) of the document referenced by {@link SpecificationReference},
+     * such as '6.2.1'.
+     *
+     * @return a document section identifier
+     */
+    String section() default "";
+
+    /**
+     * A quotation of relevant text from the section referenced by {@link #section()}.
+     *
+     * @return human-readable text from the references document and section.
+     */
+    String quote() default "";
 }

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SpecificationReference.java

@@ -0,0 +1,41 @@
+/**
+ *
+ * Copyright 2024 Guus der Kinderen
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.igniterealtime.smack.inttest.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Reference to a specific part of a specification.
+ *
+ * @author Guus der Kinderen, guus@goodbytes.nl
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface SpecificationReference {
+
+    /**
+     * Unique identifier for a specification document, such as 'RFC 6120' or 'XEP-0485'.
+     *
+     * @return a document identifier
+     */
+    String document();
+}

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/package-info.java

@@ -136,6 +136,14 @@
  * <td>List of disabled tests</td>
  * </tr>
  * <tr>
+ * <td>enabledSpecifications</td>
+ * <td>List of specifications for which to enable tests</td>
+ * </tr>
+ * <tr>
+ * <td>disabledSpecifications</td>
+ * <td>List of specificatinos for which to disable tests</td>
+ * </tr>
+ * <tr>
  * <td>defaultConnection</td>
  * <td>Nickname of the default connection</td>
  * </tr>
@@ -187,6 +195,20 @@
  * <p>
  * would run all tests defined in the <code>SoftwareInfoIntegrationTest</code> class.
  * </p>
+ * <p>
+ * Use <code>enabledSpecifications</code> to run all tests that assert implementation of functionality that is described
+ * in standards identified by the provided specification-reference.
+ * </p>
+ * <p>
+ * For example:
+ * </p>
+ *
+ * <pre>
+ * $ gradle integrationTest -Dsinttest.enabledSpecifications=XEP-0045
+ * </pre>
+ * <p>
+ * would run all tests that are annotated to verify functionality specified in XEP-0045: "Multi-User Chat".
+ * </p>
  * <h2>Overview of the components</h2>
  * <p>
  * Package <code>org.igniterealtime.smack.inttest</code>