Add workaround for truststores in JKS format when using Java >= 9

smack-core/src/main/java/org/jivesoftware/smack/util/TLSUtils.java

@@ -16,9 +16,12 @@
  */
 package org.jivesoftware.smack.util;
 
+import java.io.DataInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
 import java.security.KeyManagementException;
 import java.security.KeyStore;
 import java.security.KeyStoreException;
@@ -285,4 +288,51 @@ public class TLSUtils {
             return null;
         }
     }
+
+    enum DefaultTrustStoreType {
+        jks,
+        unknown,
+        no_default,
+    }
+
+    private static final int JKS_MAGIC = 0xfeedfeed;
+    private static final int JKS_VERSION_1 = 1;
+    private static final int JKS_VERSION_2 = 2;
+
+    public static DefaultTrustStoreType getDefaultTruststoreType() throws IOException {
+        try (InputStream inputStream = getDefaultTruststoreStreamIfPossible()) {
+            if (inputStream == null) {
+                return DefaultTrustStoreType.no_default;
+            }
+
+            DataInputStream dis = new DataInputStream(inputStream);
+            int magic = dis.readInt();
+            int version = dis.readInt();
+
+            if (magic == JKS_MAGIC && (version == JKS_VERSION_1 || version == JKS_VERSION_2)) {
+                return DefaultTrustStoreType.jks;
+            }
+        }
+
+        return DefaultTrustStoreType.unknown;
+    }
+
+    /**
+     * Tries to determine if the default truststore type is of type jks and sets the javax.net.ssl.trustStoreType system
+     * property to 'JKS' if so. This is meant as workaround in situations where the default truststore type is (still)
+     * 'jks' but we run on a newer JRE/JDK which uses PKCS#12 as type. See for example <a href="https://bugs.gentoo.org/712290">Gentoo bug #712290</a>.
+     */
+    public static void setDefaultTrustStoreTypeToJksIfRequired() {
+        DefaultTrustStoreType defaultTrustStoreType;
+        try {
+            defaultTrustStoreType = getDefaultTruststoreType();
+        } catch (IOException e) {
+            LOGGER.log(Level.WARNING, "Could not set keystore type to jks if required", e);
+            return;
+        }
+
+        if (defaultTrustStoreType == DefaultTrustStoreType.jks) {
+            System.setProperty("javax.net.ssl.trustStoreType", "JKS");
+        }
+    }
 }

smack-integration-test/src/main/java/org/igniterealtime/smack/inttest/SmackIntegrationTestFramework.java

@@ -1,6 +1,6 @@
 /**
  *
- * Copyright 2015-2019 Florian Schmaus
+ * Copyright 2015-2020 Florian Schmaus
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -56,6 +56,7 @@ import org.jivesoftware.smack.XMPPException;
 import org.jivesoftware.smack.tcp.XMPPTCPConnection;
 import org.jivesoftware.smack.tcp.XMPPTCPConnectionConfiguration;
 import org.jivesoftware.smack.util.StringUtils;
+import org.jivesoftware.smack.util.TLSUtils;
 
 import org.jivesoftware.smackx.debugger.EnhancedDebuggerWindow;
 import org.jivesoftware.smackx.iqregister.AccountManager;
@@ -71,6 +72,10 @@ import org.reflections.scanners.TypeAnnotationsScanner;
 
 public class SmackIntegrationTestFramework<DC extends AbstractXMPPConnection> {
 
+    static {
+        TLSUtils.setDefaultTrustStoreTypeToJksIfRequired();
+    }
+
     private static final Logger LOGGER = Logger.getLogger(SmackIntegrationTestFramework.class.getName());
 
     public static boolean SINTTEST_UNIT_TEST = false;