Merge pull request #588 from guusdk/sint_configurable-testresultprocessor

[sinttest] Allow custom processing of test run result

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/Configuration.java

@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.lang.reflect.Method;
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -31,6 +32,7 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 import javax.net.ssl.SSLContext;
 
@@ -131,6 +133,8 @@ public final class Configuration {
 
     public final CompatibilityMode compatibilityMode;
 
+    public final List<? extends SmackIntegrationTestFramework.TestRunResultProcessor> testRunResultProcessors;
+
     private Configuration(Configuration.Builder builder) throws KeyManagementException, NoSuchAlgorithmException {
         service = Objects.requireNonNull(builder.service,
                         "'service' must be set. Either via 'properties' files or via system property 'sinttest.service'.");
@@ -203,6 +207,7 @@ public final class Configuration {
 
         this.dnsResolver = builder.dnsResolver;
         this.compatibilityMode = builder.compatibilityMode;
+        this.testRunResultProcessors = builder.testRunResultProcessors;
     }
 
     public boolean isAccountRegistrationPossible() {
@@ -263,6 +268,8 @@ public final class Configuration {
 
         private CompatibilityMode compatibilityMode = CompatibilityMode.standardsCompliant;
 
+        private List<? extends SmackIntegrationTestFramework.TestRunResultProcessor> testRunResultProcessors;
+
         private Builder() {
         }
 
@@ -473,6 +480,15 @@ public final class Configuration {
             return setCompatibilityMode(compatibilityMode);
         }
 
+        public Builder setTestRunResultProcessors(String testRunResultProcessorsString) {
+            if (testRunResultProcessorsString == null) {
+                return this;
+            }
+
+            testRunResultProcessors = getTestRunProcessorListFrom(testRunResultProcessorsString);
+            return this;
+        }
+
         public Configuration build() throws KeyManagementException, NoSuchAlgorithmException {
             return new Configuration(this);
         }
@@ -550,6 +566,9 @@ public final class Configuration {
 
         builder.setCompatibilityMode(properties.getProperty("compatibilityMode"));
 
+        builder.setTestRunResultProcessors(properties.getProperty("testRunResultProcessors",
+            SmackIntegrationTestFramework.JulTestRunResultProcessor.class.getName()));
+
         return builder.build();
     }
 
@@ -605,6 +624,19 @@ public final class Configuration {
         return split(input, Configuration::normalizeSpecification);
     }
 
+    private static List<? extends SmackIntegrationTestFramework.TestRunResultProcessor> getTestRunProcessorListFrom(String input) {
+        return Arrays.stream(input.split(","))
+            .map(element -> {
+                try {
+                    final Class<? extends SmackIntegrationTestFramework.TestRunResultProcessor> aClass = Class.forName(element).asSubclass(SmackIntegrationTestFramework.TestRunResultProcessor.class);
+                    return aClass.getConstructor().newInstance();
+                } catch (Exception e) {
+                    throw new IllegalArgumentException("Unable to construct TestRunResultProcessor from value: " + element, e);
+                }
+            })
+            .collect(Collectors.toList());
+    }
+
     private static Map<String, Set<String>> convertTestsToMap(Set<String> tests) {
         Map<String, Set<String>> res = new HashMap<>();
         for (String test : tests) {

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/SmackIntegrationTestFramework.java

@@ -114,54 +114,65 @@ public class SmackIntegrationTestFramework {
         SmackIntegrationTestFramework sinttest = new SmackIntegrationTestFramework(config);
         TestRunResult testRunResult = sinttest.run();
 
-        for (Map.Entry<Class<? extends AbstractSmackIntTest>, Throwable> entry : testRunResult.impossibleTestClasses.entrySet()) {
+        for (final TestRunResultProcessor testRunResultProcessor : config.testRunResultProcessors) {
-            LOGGER.info("Could not run " + entry.getKey().getName() + " because: "
+            try {
-                            + entry.getValue().getLocalizedMessage());
+                testRunResultProcessor.process(testRunResult);
-        }
+            } catch (Throwable t) {
-        for (TestNotPossible testNotPossible : testRunResult.impossibleIntegrationTests) {
+                LOGGER.log(Level.WARNING, "Invocation of TestRunResultProcessor " + testRunResultProcessor + " failed.", t);
-            LOGGER.info("Could not run " + testNotPossible.concreteTest + " because: "
-                            + testNotPossible.testNotPossibleException.getMessage());
-        }
-        for (SuccessfulTest successfulTest : testRunResult.successfulIntegrationTests) {
-            LOGGER.info(successfulTest.concreteTest + " ✔");
-        }
-        final int successfulTests = testRunResult.successfulIntegrationTests.size();
-        final int failedTests = testRunResult.failedIntegrationTests.size();
-        final int availableTests = testRunResult.getNumberOfAvailableTests();
-        LOGGER.info("SmackIntegrationTestFramework[" + testRunResult.testRunId + ']' + " finished: "
-                        + successfulTests + '/' + availableTests + " [" + failedTests + " failed]");
-
-        final int exitStatus;
-        if (failedTests > 0) {
-            LOGGER.warning("💀 The following " + failedTests + " tests failed! 💀");
-            final SortedSet<String> bySpecification = new TreeSet<>();
-            for (FailedTest failedTest : testRunResult.failedIntegrationTests) {
-                final Throwable cause = failedTest.failureReason;
-                LOGGER.log(Level.SEVERE, failedTest.concreteTest + " failed: " + cause, cause);
-                if (failedTest.concreteTest.method.getDeclaringClass().isAnnotationPresent(SpecificationReference.class)) {
-                    final String specificationReference = getSpecificationReference(failedTest.concreteTest.method);
-                    if (specificationReference != null) {
-                        bySpecification.add("- " + specificationReference + " (as tested by '" + failedTest.concreteTest + "')");
-                    }
-                }
             }
-            if (!bySpecification.isEmpty()) {
-                LOGGER.log(Level.SEVERE, "The failed tests correspond to the following specifications:" + System.lineSeparator() + String.join(System.lineSeparator(), bySpecification));
-            }
-
-            exitStatus = 2;
-        } else {
-            LOGGER.info("All possible Smack Integration Tests completed successfully. \\o/");
-            exitStatus = 0;
         }
 
         if (config.debuggerFactory instanceof EnhancedDebugger) {
             EnhancedDebuggerWindow.getInstance().waitUntilClosed();
         }
 
+        final int exitStatus = testRunResult.failedIntegrationTests.isEmpty() ? 0 : 2;
         System.exit(exitStatus);
     }
 
+    public static class JulTestRunResultProcessor implements TestRunResultProcessor {
+
+        @Override
+        public void process(final TestRunResult testRunResult) {
+            for (Map.Entry<Class<? extends AbstractSmackIntTest>, Throwable> entry : testRunResult.impossibleTestClasses.entrySet()) {
+                LOGGER.info("Could not run " + entry.getKey().getName() + " because: "
+                    + entry.getValue().getLocalizedMessage());
+            }
+            for (TestNotPossible testNotPossible : testRunResult.impossibleIntegrationTests) {
+                LOGGER.info("Could not run " + testNotPossible.concreteTest + " because: "
+                    + testNotPossible.testNotPossibleException.getMessage());
+            }
+            for (SuccessfulTest successfulTest : testRunResult.successfulIntegrationTests) {
+                LOGGER.info(successfulTest.concreteTest + " ✔");
+            }
+            final int successfulTests = testRunResult.successfulIntegrationTests.size();
+            final int failedTests = testRunResult.failedIntegrationTests.size();
+            final int availableTests = testRunResult.getNumberOfAvailableTests();
+            LOGGER.info("SmackIntegrationTestFramework[" + testRunResult.testRunId + ']' + " finished: "
+                + successfulTests + '/' + availableTests + " [" + failedTests + " failed]");
+
+            if (failedTests > 0) {
+                LOGGER.warning("💀 The following " + failedTests + " tests failed! 💀");
+                final SortedSet<String> bySpecification = new TreeSet<>();
+                for (FailedTest failedTest : testRunResult.failedIntegrationTests) {
+                    final Throwable cause = failedTest.failureReason;
+                    LOGGER.log(Level.SEVERE, failedTest.concreteTest + " failed: " + cause, cause);
+                if (failedTest.concreteTest.method.getDeclaringClass().isAnnotationPresent(SpecificationReference.class)) {
+                        final String specificationReference = getSpecificationReference(failedTest.concreteTest.method);
+                        if (specificationReference != null) {
+                            bySpecification.add("- " + specificationReference + " (as tested by '" + failedTest.concreteTest + "')");
+                        }
+                    }
+                }
+                if (!bySpecification.isEmpty()) {
+                    LOGGER.log(Level.SEVERE, "The failed tests correspond to the following specifications:" + System.lineSeparator() + String.join(System.lineSeparator(), bySpecification));
+                }
+            } else {
+                LOGGER.info("All possible Smack Integration Tests completed successfully. \\o/");
+            }
+        }
+    }
+
     private static String getSpecificationReference(Method method) {
         final SpecificationReference spec = method.getDeclaringClass().getAnnotation(SpecificationReference.class);
         if (spec == null || spec.document().isBlank()) {
@@ -665,6 +676,11 @@ public class SmackIntegrationTestFramework {
         return (Exception) e;
     }
 
+    @FunctionalInterface
+    public interface TestRunResultProcessor {
+        void process(SmackIntegrationTestFramework.TestRunResult testRunResult);
+    }
+
     public static final class TestRunResult {
 
         /**

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/package-info.java

@@ -167,6 +167,10 @@
  * <td>dnsResolver</td>
  * <td>One of ‘minidns’, ‘javax’ or ‘dnsjava’. Defaults to ‘minidns’.</td>
  * </tr>
+ * <tr>
+ * <td>testRunResultProcessors</td>
+ * <td>List of class names for generating test run output. Defaults to 'org.igniterealtime.smack.inttest.SmackIntegrationTestFramework$ConsoleTestRunResultProcessor'</td>
+ * </tr>
  * </table>
  * <h3>Where to place the properties file</h3>
  * <p>
@@ -338,5 +342,18 @@
  * <pre>{@code
  * $ gradle integrationTest -Dsinttest.service=my.xmppservice.org -Dsinttest.testPackages=org.mypackage,org.otherpackage
  * }</pre>
+ * <h2>Generating test run reports</h2>
+ * <p>
+ * By default, the results of the test run is printed to standard-error. You can, however, provide your own processing
+ * for the test run results. To do so, create an implementation of
+ * {@link org.igniterealtime.smack.inttest.SmackIntegrationTestFramework.TestRunResultProcessor} and provide its class
+ * name to the <code>testRunResultProcessor</code> property. This property takes a comma separated list of class names.
+ * </p>
+ * <p>
+ * Example:
+ * </p>
+ * <pre>{@code
+ * $ gradle integrationTest -Dsinttest.service=my.xmppservice.org -Dsinttest.testRunResultProcessor=org.igniterealtime.smack.inttest.SmackIntegrationTestFramework$ConsoleTestRunResultProcessor
+ * }</pre>
  */
 package org.igniterealtime.smack.inttest;