From 69f84f24b60006c975c77e84a2de3162cf014653 Mon Sep 17 00:00:00 2001 From: Paul Schaub Date: Wed, 4 May 2022 20:55:29 +0200 Subject: [PATCH] Implement heavy duty packet inspection to figure out nature of data --- .../DecryptionStreamFactory.java | 2 +- .../OpenPgpInputStream.java | 320 ++++++++++++++++-- .../OpenPgpInputStreamTest.java | 32 +- 3 files changed, 328 insertions(+), 26 deletions(-) diff --git a/pgpainless-core/src/main/java/org/pgpainless/decryption_verification/DecryptionStreamFactory.java b/pgpainless-core/src/main/java/org/pgpainless/decryption_verification/DecryptionStreamFactory.java index a8847c33..2da18c00 100644 --- a/pgpainless-core/src/main/java/org/pgpainless/decryption_verification/DecryptionStreamFactory.java +++ b/pgpainless-core/src/main/java/org/pgpainless/decryption_verification/DecryptionStreamFactory.java @@ -140,7 +140,7 @@ public final class DecryptionStreamFactory { return new DecryptionStream(pgpInStream, resultBuilder, integrityProtectedEncryptedInputStream, null); } - if (openPgpIn.isBinaryOpenPgp()) { + if (openPgpIn.isLikelyOpenPgpMessage()) { outerDecodingStream = openPgpIn; objectFactory = ImplementationFactory.getInstance().getPGPObjectFactory(outerDecodingStream); // Parse OpenPGP message diff --git a/pgpainless-core/src/main/java/org/pgpainless/decryption_verification/OpenPgpInputStream.java b/pgpainless-core/src/main/java/org/pgpainless/decryption_verification/OpenPgpInputStream.java index d15f2ffb..fa954f97 100644 --- a/pgpainless-core/src/main/java/org/pgpainless/decryption_verification/OpenPgpInputStream.java +++ b/pgpainless-core/src/main/java/org/pgpainless/decryption_verification/OpenPgpInputStream.java @@ -4,14 +4,46 @@ package org.pgpainless.decryption_verification; +import static org.bouncycastle.bcpg.PacketTags.COMPRESSED_DATA; +import static org.bouncycastle.bcpg.PacketTags.EXPERIMENTAL_1; +import static org.bouncycastle.bcpg.PacketTags.EXPERIMENTAL_2; +import static org.bouncycastle.bcpg.PacketTags.EXPERIMENTAL_3; +import static org.bouncycastle.bcpg.PacketTags.EXPERIMENTAL_4; +import static org.bouncycastle.bcpg.PacketTags.LITERAL_DATA; +import static org.bouncycastle.bcpg.PacketTags.MARKER; +import static org.bouncycastle.bcpg.PacketTags.MOD_DETECTION_CODE; +import static org.bouncycastle.bcpg.PacketTags.ONE_PASS_SIGNATURE; +import static org.bouncycastle.bcpg.PacketTags.PUBLIC_KEY; +import static org.bouncycastle.bcpg.PacketTags.PUBLIC_KEY_ENC_SESSION; +import static org.bouncycastle.bcpg.PacketTags.PUBLIC_SUBKEY; +import static org.bouncycastle.bcpg.PacketTags.RESERVED; +import static org.bouncycastle.bcpg.PacketTags.SECRET_KEY; +import static org.bouncycastle.bcpg.PacketTags.SECRET_SUBKEY; +import static org.bouncycastle.bcpg.PacketTags.SIGNATURE; +import static org.bouncycastle.bcpg.PacketTags.SYMMETRIC_KEY_ENC; +import static org.bouncycastle.bcpg.PacketTags.SYMMETRIC_KEY_ENC_SESSION; +import static org.bouncycastle.bcpg.PacketTags.SYM_ENC_INTEGRITY_PRO; +import static org.bouncycastle.bcpg.PacketTags.TRUST; +import static org.bouncycastle.bcpg.PacketTags.USER_ATTRIBUTE; +import static org.bouncycastle.bcpg.PacketTags.USER_ID; + import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; -import org.bouncycastle.openpgp.PGPObjectFactory; -import org.pgpainless.implementation.ImplementationFactory; +import org.bouncycastle.bcpg.BCPGInputStream; +import org.bouncycastle.openpgp.PGPCompressedData; +import org.bouncycastle.openpgp.PGPEncryptedData; +import org.bouncycastle.openpgp.PGPLiteralData; +import org.bouncycastle.openpgp.PGPOnePassSignature; +import org.pgpainless.algorithm.CompressionAlgorithm; +import org.pgpainless.algorithm.HashAlgorithm; +import org.pgpainless.algorithm.PublicKeyAlgorithm; +import org.pgpainless.algorithm.SignatureType; +import org.pgpainless.algorithm.StreamEncoding; +import org.pgpainless.algorithm.SymmetricKeyAlgorithm; public class OpenPgpInputStream extends BufferedInputStream { @@ -25,8 +57,9 @@ public class OpenPgpInputStream extends BufferedInputStream { private boolean containsArmorHeader; private boolean containsOpenPgpPackets; + private boolean isLikelyOpenPgpMessage; - public OpenPgpInputStream(InputStream in) throws IOException { + public OpenPgpInputStream(InputStream in, boolean check) throws IOException { super(in, MAX_BUFFER_SIZE); mark(MAX_BUFFER_SIZE); @@ -34,10 +67,16 @@ public class OpenPgpInputStream extends BufferedInputStream { bufferLen = read(buffer); reset(); - inspectBuffer(); + if (check) { + inspectBuffer(); + } } - private void inspectBuffer() { + public OpenPgpInputStream(InputStream in) throws IOException { + this(in, true); + } + + private void inspectBuffer() throws IOException { if (determineIsArmored()) { return; } @@ -61,32 +100,250 @@ public class OpenPgpInputStream extends BufferedInputStream { * This breaks down though if we read plausible garbage where the data accidentally makes sense, * or valid, yet incomplete packets (remember, we are still only working on a portion of the data). */ - private void determineIsBinaryOpenPgp() { + private void determineIsBinaryOpenPgp() throws IOException { if (bufferLen == -1) { // Empty data return; } - try { - ByteArrayInputStream bufferIn = new ByteArrayInputStream(buffer, 0, bufferLen); - PGPObjectFactory objectFactory = ImplementationFactory.getInstance().getPGPObjectFactory(bufferIn); + ByteArrayInputStream bufferIn = new ByteArrayInputStream(buffer, 0, bufferLen); + nonExhaustiveParseAndCheckPlausibility(bufferIn); + } - boolean containsPackets = false; - while (objectFactory.nextObject() != null) { - containsPackets = true; - // read all packets in buffer - hope to confirm invalid data via thrown IOExceptions + private void nonExhaustiveParseAndCheckPlausibility(ByteArrayInputStream bufferIn) throws IOException { + int hdr = bufferIn.read(); + if (hdr < 0 || (hdr & 0x80) == 0) { + return; + } + + boolean newPacket = (hdr & 0x40) != 0; + int tag = 0; + int bodyLen = 0; + boolean partial = false; + + if (newPacket) { + tag = hdr & 0x3f; + + int l = bufferIn.read(); + if (l < 192) { + bodyLen = l; + } else if (l <= 223) { + int b = bufferIn.read(); + bodyLen = ((l - 192) << 8) + (b) + 192; + } else if (l == 255) { + bodyLen = (bufferIn.read() << 24) | (bufferIn.read() << 16) | (bufferIn.read() << 8) | bufferIn.read(); + } else { + partial = true; + bodyLen = 1 << (l & 0x1f); } - containsOpenPgpPackets = containsPackets; + } else { + int lengthType = hdr & 0x3; + tag = (hdr & 0x3f) >> 2; + switch (lengthType) { + case 0: + bodyLen = bufferIn.read(); + break; + case 1: + bodyLen = (bufferIn.read() << 8) | bufferIn.read(); + break; + case 2: + bodyLen = (bufferIn.read() << 24) | (bufferIn.read() << 16) | (bufferIn.read() << 8) | bufferIn.read(); + break; + case 3: + partial = true; + break; + default: + return; + } + } - } catch (IOException e) { - String msg = e.getMessage(); + if (bodyLen < 0) { + return; + } - // If true, we *probably* hit valid, but large OpenPGP data (not sure though) :/ - // Otherwise we hit garbage and can be sure that this is no OpenPGP data \o/ - containsOpenPgpPackets = (msg != null && msg.contains("premature end of stream in PartialInputStream")); + BCPGInputStream bcpgIn = new BCPGInputStream(bufferIn); + switch (tag) { + case RESERVED: + // How to handle this? Probably discard as garbage... + return; - // This is not an optimal way of determining the nature of data, but probably the best - // we can do :/ + case PUBLIC_KEY_ENC_SESSION: + int pkeskVersion = bcpgIn.read(); + if (pkeskVersion <= 0 || pkeskVersion > 5) { + return; + } + + // Skip Key-ID + for (int i = 0; i < 8; i++) { + bcpgIn.read(); + } + + int pkeskAlg = bcpgIn.read(); + if (PublicKeyAlgorithm.fromId(pkeskAlg) == null) { + return; + } + + containsOpenPgpPackets = true; + isLikelyOpenPgpMessage = true; + break; + + case SIGNATURE: + int sigVersion = bcpgIn.read(); + int sigType; + if (sigVersion == 2 || sigVersion == 3) { + int l = bcpgIn.read(); + sigType = bcpgIn.read(); + } else if (sigVersion == 4 || sigVersion == 5) { + sigType = bcpgIn.read(); + } else { + return; + } + + try { + SignatureType.valueOf(sigType); + } catch (IllegalArgumentException e) { + return; + } + + containsOpenPgpPackets = true; + isLikelyOpenPgpMessage = true; + break; + + case SYMMETRIC_KEY_ENC_SESSION: + int skeskVersion = bcpgIn.read(); + if (skeskVersion == 4) { + int skeskAlg = bcpgIn.read(); + if (SymmetricKeyAlgorithm.fromId(skeskAlg) == null) { + return; + } + // TODO: Parse S2K? + } else { + return; + } + containsOpenPgpPackets = true; + isLikelyOpenPgpMessage = true; + break; + + case ONE_PASS_SIGNATURE: + int opsVersion = bcpgIn.read(); + if (opsVersion == 3) { + int opsSigType = bcpgIn.read(); + try { + SignatureType.valueOf(opsSigType); + } catch (IllegalArgumentException e) { + return; + } + int opsHashAlg = bcpgIn.read(); + if (HashAlgorithm.fromId(opsHashAlg) == null) { + return; + } + int opsKeyAlg = bcpgIn.read(); + if (PublicKeyAlgorithm.fromId(opsKeyAlg) == null) { + return; + } + } else { + return; + } + + containsOpenPgpPackets = true; + isLikelyOpenPgpMessage = true; + break; + + case SECRET_KEY: + case PUBLIC_KEY: + case SECRET_SUBKEY: + case PUBLIC_SUBKEY: + int keyVersion = bcpgIn.read(); + for (int i = 0; i < 4; i++) { + // Creation time + bcpgIn.read(); + } + if (keyVersion == 3) { + long validDays = (in.read() << 8) | in.read(); + if (validDays < 0) { + return; + } + } else if (keyVersion == 4) { + + } else if (keyVersion == 5) { + + } else { + return; + } + int keyAlg = bcpgIn.read(); + if (PublicKeyAlgorithm.fromId(keyAlg) == null) { + return; + } + + containsOpenPgpPackets = true; + break; + + case COMPRESSED_DATA: + int compAlg = bcpgIn.read(); + if (CompressionAlgorithm.fromId(compAlg) == null) { + return; + } + + containsOpenPgpPackets = true; + isLikelyOpenPgpMessage = true; + break; + + case SYMMETRIC_KEY_ENC: + // No data to compare :( + containsOpenPgpPackets = true; + break; + + case MARKER: + byte[] marker = new byte[3]; + bcpgIn.readFully(marker); + if (marker[0] != 0x50 || marker[1] != 0x47 || marker[2] != 0x50) { + return; + } + + containsOpenPgpPackets = true; + break; + + case LITERAL_DATA: + int format = bcpgIn.read(); + if (StreamEncoding.fromCode(format) == null) { + return; + } + + containsOpenPgpPackets = true; + isLikelyOpenPgpMessage = true; + break; + + case TRUST: + case USER_ID: + case USER_ATTRIBUTE: + // Not much to compare + containsOpenPgpPackets = true; + break; + + case SYM_ENC_INTEGRITY_PRO: + int seipVersion = bcpgIn.read(); + if (seipVersion != 1) { + return; + } + isLikelyOpenPgpMessage = true; + containsOpenPgpPackets = true; + break; + + case MOD_DETECTION_CODE: + byte[] digest = new byte[20]; + bcpgIn.readFully(digest); + + containsOpenPgpPackets = true; + break; + + case EXPERIMENTAL_1: + case EXPERIMENTAL_2: + case EXPERIMENTAL_3: + case EXPERIMENTAL_4: + return; + default: + containsOpenPgpPackets = false; + break; } } @@ -119,10 +376,31 @@ public class OpenPgpInputStream extends BufferedInputStream { return containsArmorHeader; } + /** + * Return true, if the data is possibly binary OpenPGP. + * The criterion for this are less strict than for {@link #isLikelyOpenPgpMessage()}, + * as it also accepts other OpenPGP packets at the beginning of the data stream. + * + * Use with caution. + * + * @return true if data appears to be binary OpenPGP data + */ public boolean isBinaryOpenPgp() { return containsOpenPgpPackets; } + /** + * Returns true, if the underlying data is very likely (more than 99,9%) an OpenPGP message. + * OpenPGP Message means here that it starts with either an {@link PGPEncryptedData}, + * {@link PGPCompressedData}, {@link PGPOnePassSignature} or {@link PGPLiteralData} packet. + * The plausability of these data packets is checked as far as possible. + * + * @return true if likely OpenPGP message + */ + public boolean isLikelyOpenPgpMessage() { + return isLikelyOpenPgpMessage; + } + public boolean isNonOpenPgp() { return !isAsciiArmored() && !isBinaryOpenPgp(); } diff --git a/pgpainless-core/src/test/java/org/pgpainless/decryption_verification/OpenPgpInputStreamTest.java b/pgpainless-core/src/test/java/org/pgpainless/decryption_verification/OpenPgpInputStreamTest.java index f416ea7f..67719886 100644 --- a/pgpainless-core/src/test/java/org/pgpainless/decryption_verification/OpenPgpInputStreamTest.java +++ b/pgpainless-core/src/test/java/org/pgpainless/decryption_verification/OpenPgpInputStreamTest.java @@ -11,18 +11,22 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.Random; import org.bouncycastle.bcpg.ArmoredInputStream; +import org.bouncycastle.bcpg.CompressionAlgorithmTags; +import org.bouncycastle.openpgp.PGPCompressedDataGenerator; import org.bouncycastle.util.io.Streams; +import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; public class OpenPgpInputStreamTest { private static final Random RANDOM = new Random(); - @Test + @RepeatedTest(10) public void randomBytesDoNotContainOpenPgpData() throws IOException { byte[] randomBytes = new byte[1000000]; RANDOM.nextBytes(randomBytes); @@ -30,13 +34,33 @@ public class OpenPgpInputStreamTest { OpenPgpInputStream openPgpInputStream = new OpenPgpInputStream(randomIn); assertFalse(openPgpInputStream.isAsciiArmored()); - assertFalse(openPgpInputStream.isBinaryOpenPgp()); - assertTrue(openPgpInputStream.isNonOpenPgp()); + assertFalse(openPgpInputStream.isLikelyOpenPgpMessage()); ByteArrayOutputStream out = new ByteArrayOutputStream(); Streams.pipeAll(openPgpInputStream, out); + byte[] outBytes = out.toByteArray(); - assertArrayEquals(randomBytes, out.toByteArray()); + assertArrayEquals(randomBytes, outBytes); + } + + @RepeatedTest(10) + public void largeCompressedDataIsBinaryOpenPgp() throws IOException { + // Since we are compressing RANDOM data, the output will likely be roughly the same size + // So we very likely will end up with data larger than the MAX_BUFFER_SIZE + byte[] randomBytes = new byte[OpenPgpInputStream.MAX_BUFFER_SIZE * 10]; + RANDOM.nextBytes(randomBytes); + + ByteArrayOutputStream compressedDataPacket = new ByteArrayOutputStream(); + PGPCompressedDataGenerator compressedDataGenerator = new PGPCompressedDataGenerator(CompressionAlgorithmTags.ZIP); + OutputStream compressor = compressedDataGenerator.open(compressedDataPacket); + compressor.write(randomBytes); + compressor.close(); + + OpenPgpInputStream inputStream = new OpenPgpInputStream(new ByteArrayInputStream(compressedDataPacket.toByteArray())); + assertFalse(inputStream.isAsciiArmored()); + assertFalse(inputStream.isNonOpenPgp()); + assertTrue(inputStream.isBinaryOpenPgp()); + assertTrue(inputStream.isLikelyOpenPgpMessage()); } @Test