diff --git a/pgpainless-core/src/main/java/org/pgpainless/util/Passphrase.java b/pgpainless-core/src/main/java/org/pgpainless/util/Passphrase.java
index 2168b0da..a46745df 100644
--- a/pgpainless-core/src/main/java/org/pgpainless/util/Passphrase.java
+++ b/pgpainless-core/src/main/java/org/pgpainless/util/Passphrase.java
@@ -98,7 +98,7 @@ public class Passphrase {
      */
     public boolean isEmpty() {
         synchronized (lock) {
-            return chars == null;
+            return valid && chars == null;
         }
     }