vanitasvitae/Smack
1
0
Fork 0
mirror of https://github.com/vanitasvitae/Smack.git synced 2024-11-22 03:52:06 +01:00
Code Issues Releases Wiki Activity

[sinttest] Add tagging of tests with references to (XMPP) specifications

Browse source 
A new annotation is introduced (`SpecificationReference`) that can be used to annotate a SINT test class

The properties are available in the annotation:
- `document`: Identifier for a specification document, such as 'RFC 6120' or 'XEP-0485'

The pre-existing `SmackIntegrationTest` annotation has now received two new properties:
- `section`: Identifier for a section (or paragraph), such as '6.2.1'
- `quote`: A quotation of relevant text from the section
These are expected to be used in context of the `SpecificationReference` annotation.

The SINT execution framework is modified so that two new configuration options are available:
- `enabledSpecifications`
- `disabledSpecifications`

These operate on the value of the `document` property of the annotation. Their usage is comparable
to that of the pre-existing `enabledTests` and `disabledTest` configuration options.

Execution output now includes the document, section and quote that's on the annotated test, when
the test fails. This allows an end-user to easily correspond a test failure with a particular
specification.
This commit is contained in:
Guus der Kinderen 2024-03-14 14:01:21 +01:00
parent e504bc23cf
commit f76f0791e6
5 changed files with 185 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/Configuration.java
View file

@ -111,6 +111,10 @@ public final class Configuration {
private final Map<String, Set<String>> disabledTestsMap;
public final Set<String> enabledSpecifications;
public final Set<String> disabledSpecifications;
public final String defaultConnectionNickname;
public final Set<String> enabledConnections;
@ -181,6 +185,8 @@ public final class Configuration {
this.enabledTestsMap = convertTestsToMap(enabledTests);
this.disabledTests = CollectionUtil.nullSafeUnmodifiableSet(builder.disabledTests);
this.disabledTestsMap = convertTestsToMap(disabledTests);
this.enabledSpecifications = CollectionUtil.nullSafeUnmodifiableSet(builder.enabledSpecifications);
this.disabledSpecifications = CollectionUtil.nullSafeUnmodifiableSet(builder.disabledSpecifications);
this.defaultConnectionNickname = builder.defaultConnectionNickname;
this.enabledConnections = builder.enabledConnections;
this.disabledConnections = builder.disabledConnections;
@ -252,6 +258,10 @@ public final class Configuration {
private Set<String> disabledTests;
private Set<String> enabledSpecifications;
private Set<String> disabledSpecifications;
private String defaultConnectionNickname;
private Set<String> enabledConnections;
@ -378,6 +388,16 @@ public final class Configuration {
return this;
}
public Builder setEnabledSpecifications(String enabledSpecificationsString) {
enabledSpecifications = getSpecificationSetFrom(enabledSpecificationsString);
return this;
}
public Builder setDisabledSpecifications(String disabledSpecificationsString) {
disabledSpecifications = getSpecificationSetFrom(disabledSpecificationsString);
return this;
}
public Builder setDefaultConnection(String defaultConnectionNickname) {
this.defaultConnectionNickname = defaultConnectionNickname;
return this;
@ -523,6 +543,8 @@ public final class Configuration {
builder.setDebugger(properties.getProperty("debugger"));
builder.setEnabledTests(properties.getProperty("enabledTests"));
builder.setDisabledTests(properties.getProperty("disabledTests"));
builder.setEnabledSpecifications(properties.getProperty("enabledSpecifications"));
builder.setDisabledSpecifications(properties.getProperty("disabledSpecifications"));
builder.setDefaultConnection(properties.getProperty("defaultConnection"));
builder.setEnabledConnections(properties.getProperty("enabledConnections"));
builder.setDisabledConnections(properties.getProperty("disabledConnections"));
@ -587,6 +609,10 @@ public final class Configuration {
});
}
private static Set<String> getSpecificationSetFrom(String input) {
return split(input, Configuration::normalizeSpecification);
}
private static Map<String, Set<String>> convertTestsToMap(Set<String> tests) {
Map<String, Set<String>> res = new HashMap<>();
for (String test : tests) {
@ -695,4 +721,34 @@ public final class Configuration {
return contains(method, disabledTestsMap);
}
public boolean isSpecificationEnabled(String specification) {
if (enabledSpecifications.isEmpty()) {
return true;
}
if (specification == null) {
return false;
}
return enabledSpecifications.contains(normalizeSpecification(specification));
}
public boolean isSpecificationDisabled(String specification) {
if (disabledSpecifications.isEmpty()) {
return false;
}
if (specification == null) {
return false;
}
return disabledSpecifications.contains(normalizeSpecification(specification));
}
static String normalizeSpecification(String specification) {
if (specification == null || specification.isBlank()) {
return null;
}
return specification.replaceAll("\\s", "").toUpperCase();
}
}

52
smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/SmackIntegrationTestFramework.java
View file

@ -44,6 +44,8 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.logging.Level;
import java.util.logging.Logger;
@ -71,6 +73,7 @@ import org.igniterealtime.smack.inttest.Configuration.AccountRegistration;
import org.igniterealtime.smack.inttest.annotations.AfterClass;
import org.igniterealtime.smack.inttest.annotations.BeforeClass;
import org.igniterealtime.smack.inttest.annotations.SmackIntegrationTest;
import org.igniterealtime.smack.inttest.annotations.SpecificationReference;
import org.reflections.Reflections;
import org.reflections.scanners.MethodAnnotationsScanner;
import org.reflections.scanners.MethodParameterScanner;
@ -128,10 +131,21 @@ public class SmackIntegrationTestFramework {
final int exitStatus;
if (failedTests > 0) {
LOGGER.warning("💀 The following " + failedTests + " tests failed! 💀");
final SortedSet<String> bySpecification = new TreeSet<>();
for (FailedTest failedTest : testRunResult.failedIntegrationTests) {
final Throwable cause = failedTest.failureReason;
LOGGER.log(Level.SEVERE, failedTest.concreteTest + " failed: " + cause, cause);
if (failedTest.concreteTest.method.isAnnotationPresent(SpecificationReference.class)) {
final String specificationReference = getSpecificationReference(failedTest.concreteTest.method);
if (specificationReference != null) {
bySpecification.add("- " + specificationReference + " (as tested by '" + failedTest.concreteTest + "')");
}
}
}
if (!bySpecification.isEmpty()) {
LOGGER.log(Level.SEVERE, "The failed tests correspond to the following specifications:" + System.lineSeparator() + String.join(System.lineSeparator(), bySpecification));
}
exitStatus = 2;
} else {
LOGGER.info("All possible Smack Integration Tests completed successfully. \\o/");
@ -149,6 +163,24 @@ public class SmackIntegrationTestFramework {
System.exit(exitStatus);
}
private static String getSpecificationReference(Method method) {
final SpecificationReference spec = method.getDeclaringClass().getAnnotation(SpecificationReference.class);
if (spec == null || spec.document().isBlank()) {
return null;
}
String line = spec.document().trim();
final SmackIntegrationTest test = method.getAnnotation(SmackIntegrationTest.class);
if (!test.section().isBlank()) {
line += " section " + test.section().trim();
}
if (!test.quote().isBlank()) {
line += ":\t\"" + test.quote().trim() + "\"";
}
assert !line.isBlank();
return line;
}
public SmackIntegrationTestFramework(Configuration configuration) {
this.config = configuration;
}
@ -297,6 +329,26 @@ public class SmackIntegrationTestFramework {
continue;
}
final String specification;
if (testClass.isAnnotationPresent(SpecificationReference.class)) {
final SpecificationReference specificationReferenceAnnotation = testClass.getAnnotation(SpecificationReference.class);
specification = Configuration.normalizeSpecification(specificationReferenceAnnotation.document());
} else {
specification = null;
}
if (!config.isSpecificationEnabled(specification)) {
DisabledTestClass disabledTestClass = new DisabledTestClass(testClass, "Skipping test method " + testClass + " because it tests a specification ('" + specification + "') that is not enabled");
testRunResult.disabledTestClasses.add(disabledTestClass);
continue;
}
if (config.isSpecificationDisabled(specification)) {
DisabledTestClass disabledTestClass = new DisabledTestClass(testClass, "Skipping test method " + testClass + " because it tests a specification ('" + specification + "') that is disabled");
testRunResult.disabledTestClasses.add(disabledTestClass);
continue;
}
final Constructor<? extends AbstractSmackIntTest> cons;
try {
cons = testClass.getConstructor(SmackIntegrationTestEnvironment.class);

14
smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SmackIntegrationTest.java
View file

@ -31,4 +31,18 @@ public @interface SmackIntegrationTest {
int connectionCount() default -1;
/**
* Unique identifier for a section (or paragraph) of the document referenced by {@link SpecificationReference},
* such as '6.2.1'.
*
* @return a document section identifier
*/
String section() default "";
/**
* A quotation of relevant text from the section referenced by {@link #section()}.
*
* @return human-readable text from the references document and section.
*/
String quote() default "";
}

41
smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/annotations/SpecificationReference.java Normal file
View file

@ -0,0 +1,41 @@
/**
*
* Copyright 2024 Guus der Kinderen
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.igniterealtime.smack.inttest.annotations;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Reference to a specific part of a specification.
*
* @author Guus der Kinderen, guus@goodbytes.nl
*/
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
public @interface SpecificationReference {
/**
* Unique identifier for a specification document, such as 'RFC 6120' or 'XEP-0485'.
*
* @return a document identifier
*/
String document();
}

22
smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/package-info.java
View file

@ -136,6 +136,14 @@
* <td>List of disabled tests</td>
* </tr>
* <tr>
* <td>enabledSpecifications</td>
* <td>List of specifications for which to enable tests</td>
* </tr>
* <tr>
* <td>disabledSpecifications</td>
* <td>List of specificatinos for which to disable tests</td>
* </tr>
* <tr>
* <td>defaultConnection</td>
* <td>Nickname of the default connection</td>
* </tr>
@ -187,6 +195,20 @@
* <p>
* would run all tests defined in the <code>SoftwareInfoIntegrationTest</code> class.
* </p>
* <p>
* Use <code>enabledSpecifications</code> to run all tests that assert implementation of functionality that is described
* in standards identified by the provided specification-reference.
* </p>
* <p>
* For example:
* </p>
*
* <pre>
* $ gradle integrationTest -Dsinttest.enabledSpecifications=XEP-0045
* </pre>
* <p>
* would run all tests that are annotated to verify functionality specified in XEP-0045: "Multi-User Chat".
* </p>
* <h2>Overview of the components</h2>
* <p>
* Package <code>org.igniterealtime.smack.inttest</code>