From f76f0791e60bb4ca01fc426a28fc81a1c55102ce Mon Sep 17 00:00:00 2001 From: Guus der Kinderen Date: Thu, 14 Mar 2024 14:01:21 +0100 Subject: [PATCH] [sinttest] Add tagging of tests with references to (XMPP) specifications A new annotation is introduced (`SpecificationReference`) that can be used to annotate a SINT test class The properties are available in the annotation: - `document`: Identifier for a specification document, such as 'RFC 6120' or 'XEP-0485' The pre-existing `SmackIntegrationTest` annotation has now received two new properties: - `section`: Identifier for a section (or paragraph), such as '6.2.1' - `quote`: A quotation of relevant text from the section These are expected to be used in context of the `SpecificationReference` annotation. The SINT execution framework is modified so that two new configuration options are available: - `enabledSpecifications` - `disabledSpecifications` These operate on the value of the `document` property of the annotation. Their usage is comparable to that of the pre-existing `enabledTests` and `disabledTest` configuration options. Execution output now includes the document, section and quote that's on the annotated test, when the test fails. This allows an end-user to easily correspond a test failure with a particular specification. --- .../smack/inttest/Configuration.java | 56 +++++++++++++++++++ .../SmackIntegrationTestFramework.java | 52 +++++++++++++++++ .../annotations/SmackIntegrationTest.java | 14 +++++ .../annotations/SpecificationReference.java | 41 ++++++++++++++ .../smack/inttest/package-info.java | 22 ++++++++ 5 files changed, 185 insertions(+) create mode 100644 smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SpecificationReference.java diff --git a/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/Configuration.java b/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/Configuration.java index 0df16141c..eaf54e3ba 100644 --- a/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/Configuration.java +++ b/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/Configuration.java @@ -111,6 +111,10 @@ public final class Configuration { private final Map> disabledTestsMap; + public final Set enabledSpecifications; + + public final Set disabledSpecifications; + public final String defaultConnectionNickname; public final Set enabledConnections; @@ -181,6 +185,8 @@ public final class Configuration { this.enabledTestsMap = convertTestsToMap(enabledTests); this.disabledTests = CollectionUtil.nullSafeUnmodifiableSet(builder.disabledTests); this.disabledTestsMap = convertTestsToMap(disabledTests); + this.enabledSpecifications = CollectionUtil.nullSafeUnmodifiableSet(builder.enabledSpecifications); + this.disabledSpecifications = CollectionUtil.nullSafeUnmodifiableSet(builder.disabledSpecifications); this.defaultConnectionNickname = builder.defaultConnectionNickname; this.enabledConnections = builder.enabledConnections; this.disabledConnections = builder.disabledConnections; @@ -252,6 +258,10 @@ public final class Configuration { private Set disabledTests; + private Set enabledSpecifications; + + private Set disabledSpecifications; + private String defaultConnectionNickname; private Set enabledConnections; @@ -378,6 +388,16 @@ public final class Configuration { return this; } + public Builder setEnabledSpecifications(String enabledSpecificationsString) { + enabledSpecifications = getSpecificationSetFrom(enabledSpecificationsString); + return this; + } + + public Builder setDisabledSpecifications(String disabledSpecificationsString) { + disabledSpecifications = getSpecificationSetFrom(disabledSpecificationsString); + return this; + } + public Builder setDefaultConnection(String defaultConnectionNickname) { this.defaultConnectionNickname = defaultConnectionNickname; return this; @@ -523,6 +543,8 @@ public final class Configuration { builder.setDebugger(properties.getProperty("debugger")); builder.setEnabledTests(properties.getProperty("enabledTests")); builder.setDisabledTests(properties.getProperty("disabledTests")); + builder.setEnabledSpecifications(properties.getProperty("enabledSpecifications")); + builder.setDisabledSpecifications(properties.getProperty("disabledSpecifications")); builder.setDefaultConnection(properties.getProperty("defaultConnection")); builder.setEnabledConnections(properties.getProperty("enabledConnections")); builder.setDisabledConnections(properties.getProperty("disabledConnections")); @@ -587,6 +609,10 @@ public final class Configuration { }); } + private static Set getSpecificationSetFrom(String input) { + return split(input, Configuration::normalizeSpecification); + } + private static Map> convertTestsToMap(Set tests) { Map> res = new HashMap<>(); for (String test : tests) { @@ -695,4 +721,34 @@ public final class Configuration { return contains(method, disabledTestsMap); } + public boolean isSpecificationEnabled(String specification) { + if (enabledSpecifications.isEmpty()) { + return true; + } + + if (specification == null) { + return false; + } + + return enabledSpecifications.contains(normalizeSpecification(specification)); + } + + public boolean isSpecificationDisabled(String specification) { + if (disabledSpecifications.isEmpty()) { + return false; + } + + if (specification == null) { + return false; + } + + return disabledSpecifications.contains(normalizeSpecification(specification)); + } + + static String normalizeSpecification(String specification) { + if (specification == null || specification.isBlank()) { + return null; + } + return specification.replaceAll("\\s", "").toUpperCase(); + } } diff --git a/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/SmackIntegrationTestFramework.java b/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/SmackIntegrationTestFramework.java index 8be908fb0..63e47bf0b 100644 --- a/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/SmackIntegrationTestFramework.java +++ b/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/SmackIntegrationTestFramework.java @@ -44,6 +44,8 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; @@ -71,6 +73,7 @@ import org.igniterealtime.smack.inttest.Configuration.AccountRegistration; import org.igniterealtime.smack.inttest.annotations.AfterClass; import org.igniterealtime.smack.inttest.annotations.BeforeClass; import org.igniterealtime.smack.inttest.annotations.SmackIntegrationTest; +import org.igniterealtime.smack.inttest.annotations.SpecificationReference; import org.reflections.Reflections; import org.reflections.scanners.MethodAnnotationsScanner; import org.reflections.scanners.MethodParameterScanner; @@ -128,10 +131,21 @@ public class SmackIntegrationTestFramework { final int exitStatus; if (failedTests > 0) { LOGGER.warning("💀 The following " + failedTests + " tests failed! 💀"); + final SortedSet bySpecification = new TreeSet<>(); for (FailedTest failedTest : testRunResult.failedIntegrationTests) { final Throwable cause = failedTest.failureReason; LOGGER.log(Level.SEVERE, failedTest.concreteTest + " failed: " + cause, cause); + if (failedTest.concreteTest.method.isAnnotationPresent(SpecificationReference.class)) { + final String specificationReference = getSpecificationReference(failedTest.concreteTest.method); + if (specificationReference != null) { + bySpecification.add("- " + specificationReference + " (as tested by '" + failedTest.concreteTest + "')"); + } + } } + if (!bySpecification.isEmpty()) { + LOGGER.log(Level.SEVERE, "The failed tests correspond to the following specifications:" + System.lineSeparator() + String.join(System.lineSeparator(), bySpecification)); + } + exitStatus = 2; } else { LOGGER.info("All possible Smack Integration Tests completed successfully. \\o/"); @@ -149,6 +163,24 @@ public class SmackIntegrationTestFramework { System.exit(exitStatus); } + private static String getSpecificationReference(Method method) { + final SpecificationReference spec = method.getDeclaringClass().getAnnotation(SpecificationReference.class); + if (spec == null || spec.document().isBlank()) { + return null; + } + String line = spec.document().trim(); + + final SmackIntegrationTest test = method.getAnnotation(SmackIntegrationTest.class); + if (!test.section().isBlank()) { + line += " section " + test.section().trim(); + } + if (!test.quote().isBlank()) { + line += ":\t\"" + test.quote().trim() + "\""; + } + assert !line.isBlank(); + return line; + } + public SmackIntegrationTestFramework(Configuration configuration) { this.config = configuration; } @@ -297,6 +329,26 @@ public class SmackIntegrationTestFramework { continue; } + final String specification; + if (testClass.isAnnotationPresent(SpecificationReference.class)) { + final SpecificationReference specificationReferenceAnnotation = testClass.getAnnotation(SpecificationReference.class); + specification = Configuration.normalizeSpecification(specificationReferenceAnnotation.document()); + } else { + specification = null; + } + + if (!config.isSpecificationEnabled(specification)) { + DisabledTestClass disabledTestClass = new DisabledTestClass(testClass, "Skipping test method " + testClass + " because it tests a specification ('" + specification + "') that is not enabled"); + testRunResult.disabledTestClasses.add(disabledTestClass); + continue; + } + + if (config.isSpecificationDisabled(specification)) { + DisabledTestClass disabledTestClass = new DisabledTestClass(testClass, "Skipping test method " + testClass + " because it tests a specification ('" + specification + "') that is disabled"); + testRunResult.disabledTestClasses.add(disabledTestClass); + continue; + } + final Constructor cons; try { cons = testClass.getConstructor(SmackIntegrationTestEnvironment.class); diff --git a/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SmackIntegrationTest.java b/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SmackIntegrationTest.java index 173b4d2e9..55b1ef684 100644 --- a/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SmackIntegrationTest.java +++ b/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SmackIntegrationTest.java @@ -31,4 +31,18 @@ public @interface SmackIntegrationTest { int connectionCount() default -1; + /** + * Unique identifier for a section (or paragraph) of the document referenced by {@link SpecificationReference}, + * such as '6.2.1'. + * + * @return a document section identifier + */ + String section() default ""; + + /** + * A quotation of relevant text from the section referenced by {@link #section()}. + * + * @return human-readable text from the references document and section. + */ + String quote() default ""; } diff --git a/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SpecificationReference.java b/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SpecificationReference.java new file mode 100644 index 000000000..947dc3b90 --- /dev/null +++ b/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SpecificationReference.java @@ -0,0 +1,41 @@ +/** + * + * Copyright 2024 Guus der Kinderen + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.igniterealtime.smack.inttest.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Reference to a specific part of a specification. + * + * @author Guus der Kinderen, guus@goodbytes.nl + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface SpecificationReference { + + /** + * Unique identifier for a specification document, such as 'RFC 6120' or 'XEP-0485'. + * + * @return a document identifier + */ + String document(); +} diff --git a/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/package-info.java b/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/package-info.java index 2caf8d792..a6692a5c0 100644 --- a/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/package-info.java +++ b/smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/package-info.java @@ -136,6 +136,14 @@ * List of disabled tests * * + * enabledSpecifications + * List of specifications for which to enable tests + * + * + * disabledSpecifications + * List of specificatinos for which to disable tests + * + * * defaultConnection * Nickname of the default connection * @@ -187,6 +195,20 @@ *

* would run all tests defined in the SoftwareInfoIntegrationTest class. *

+ *

+ * Use enabledSpecifications to run all tests that assert implementation of functionality that is described + * in standards identified by the provided specification-reference. + *

+ *

+ * For example: + *

+ * + * 
+ * $ gradle integrationTest -Dsinttest.enabledSpecifications=XEP-0045
+ *
+ *

+ * would run all tests that are annotated to verify functionality specified in XEP-0045: "Multi-User Chat". + *

*

Overview of the components

*

* Package org.igniterealtime.smack.inttest