From bf7bd9a16c172bcb5ea6b24717a0429e12e2e3d1 Mon Sep 17 00:00:00 2001 From: Ferenc Rakoczi Date: Tue, 14 Nov 2023 17:00:30 +0000 Subject: [PATCH] 8317547: Enhance TLS connection support Reviewed-by: ahgross, rhalade, weijun, valeriep --- .../com/sun/crypto/provider/RSACipher.java | 22 +++-- .../classes/sun/security/util/KeyUtil.java | 59 +++++++------ .../sun/security/mscapi/CRSACipher.java | 87 +++++++++++-------- .../windows/native/libsunmscapi/security.cpp | 64 ++++++++++---- 4 files changed, 142 insertions(+), 90 deletions(-) diff --git a/src/java.base/share/classes/com/sun/crypto/provider/RSACipher.java b/src/java.base/share/classes/com/sun/crypto/provider/RSACipher.java index 4278003c465..df76f78bfb5 100644 --- a/src/java.base/share/classes/com/sun/crypto/provider/RSACipher.java +++ b/src/java.base/share/classes/com/sun/crypto/provider/RSACipher.java @@ -98,6 +98,7 @@ public final class RSACipher extends CipherSpi { // cipher parameter for OAEP padding and TLS RSA premaster secret private AlgorithmParameterSpec spec = null; + private boolean forTlsPremasterSecret = false; // buffer for the data private byte[] buffer; @@ -286,6 +287,7 @@ public final class RSACipher extends CipherSpi { } spec = params; + forTlsPremasterSecret = true; this.random = random; // for TLS RSA premaster secret } int blockType = (mode <= MODE_DECRYPT) ? RSAPadding.PAD_BLOCKTYPE_2 @@ -377,7 +379,7 @@ public final class RSACipher extends CipherSpi { byte[] decryptBuffer = RSACore.convert(buffer, 0, bufOfs); paddingCopy = RSACore.rsa(decryptBuffer, privateKey, false); result = padding.unpad(paddingCopy); - if (result == null) { + if (result == null && !forTlsPremasterSecret) { throw new BadPaddingException ("Padding error in decryption"); } @@ -466,26 +468,22 @@ public final class RSACipher extends CipherSpi { boolean isTlsRsaPremasterSecret = algorithm.equals("TlsRsaPremasterSecret"); - Exception failover = null; byte[] encoded = null; update(wrappedKey, 0, wrappedKey.length); try { encoded = doFinal(); - } catch (BadPaddingException e) { - if (isTlsRsaPremasterSecret) { - failover = e; - } else { - throw new InvalidKeyException("Unwrapping failed", e); - } - } catch (IllegalBlockSizeException e) { - // should not occur, handled with length check above + } catch (BadPaddingException | IllegalBlockSizeException e) { + // BadPaddingException cannot happen for TLS RSA unwrap. + // In that case, padding error is indicated by returning null. + // IllegalBlockSizeException cannot happen in any case, + // because of the length check above. throw new InvalidKeyException("Unwrapping failed", e); } try { if (isTlsRsaPremasterSecret) { - if (!(spec instanceof TlsRsaPremasterSecretParameterSpec)) { + if (!forTlsPremasterSecret) { throw new IllegalStateException( "No TlsRsaPremasterSecretParameterSpec specified"); } @@ -494,7 +492,7 @@ public final class RSACipher extends CipherSpi { encoded = KeyUtil.checkTlsPreMasterSecretKey( ((TlsRsaPremasterSecretParameterSpec) spec).getClientVersion(), ((TlsRsaPremasterSecretParameterSpec) spec).getServerVersion(), - random, encoded, (failover != null)); + random, encoded, encoded == null); } return ConstructKeys.constructKey(encoded, algorithm, type); diff --git a/src/java.base/share/classes/sun/security/util/KeyUtil.java b/src/java.base/share/classes/sun/security/util/KeyUtil.java index 059467779b9..c38889ed494 100644 --- a/src/java.base/share/classes/sun/security/util/KeyUtil.java +++ b/src/java.base/share/classes/sun/security/util/KeyUtil.java @@ -291,13 +291,14 @@ public final class KeyUtil { * contains the lower of that suggested by the client in the client * hello and the highest supported by the server. * @param encoded the encoded key in its "RAW" encoding format - * @param isFailOver whether the previous decryption of the - * encrypted PreMasterSecret message run into problem + * @param failure true if encoded is incorrect according to previous checks * @return the polished PreMasterSecret key in its "RAW" encoding format */ public static byte[] checkTlsPreMasterSecretKey( int clientVersion, int serverVersion, SecureRandom random, - byte[] encoded, boolean isFailOver) { + byte[] encoded, boolean failure) { + + byte[] tmp; if (random == null) { random = JCAUtil.getSecureRandom(); @@ -305,30 +306,38 @@ public final class KeyUtil { byte[] replacer = new byte[48]; random.nextBytes(replacer); - if (!isFailOver && (encoded != null)) { - // check the length - if (encoded.length != 48) { - // private, don't need to clone the byte array. - return replacer; - } - - int encodedVersion = - ((encoded[0] & 0xFF) << 8) | (encoded[1] & 0xFF); - if (clientVersion != encodedVersion) { - if (clientVersion > 0x0301 || // 0x0301: TLSv1 - serverVersion != encodedVersion) { - encoded = replacer; - } // Otherwise, For compatibility, we maintain the behavior - // that the version in pre_master_secret can be the - // negotiated version for TLS v1.0 and SSL v3.0. - } - - // private, don't need to clone the byte array. - return encoded; + if (failure) { + tmp = replacer; + } else { + tmp = encoded; } - // private, don't need to clone the byte array. - return replacer; + if (tmp == null) { + encoded = replacer; + } else { + encoded = tmp; + } + // check the length + if (encoded.length != 48) { + // private, don't need to clone the byte array. + tmp = replacer; + } else { + tmp = encoded; + } + + int encodedVersion = + ((tmp[0] & 0xFF) << 8) | (tmp[1] & 0xFF); + int check1 = 0; + int check2 = 0; + int check3 = 0; + if (clientVersion != encodedVersion) check1 = 1; + if (clientVersion > 0x0301) check2 = 1; + if (serverVersion != encodedVersion) check3 = 1; + if ((check1 & (check2 | check3)) == 1) { + return replacer; + } else { + return tmp; + } } /** diff --git a/src/jdk.crypto.mscapi/windows/classes/sun/security/mscapi/CRSACipher.java b/src/jdk.crypto.mscapi/windows/classes/sun/security/mscapi/CRSACipher.java index ef6fa63e0a6..7b2fb631023 100644 --- a/src/jdk.crypto.mscapi/windows/classes/sun/security/mscapi/CRSACipher.java +++ b/src/jdk.crypto.mscapi/windows/classes/sun/security/mscapi/CRSACipher.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,6 +30,7 @@ import java.security.*; import java.security.Key; import java.security.interfaces.*; import java.security.spec.*; +import java.util.Arrays; import javax.crypto.*; import javax.crypto.spec.*; @@ -61,6 +62,9 @@ import sun.security.util.KeyUtil; */ public final class CRSACipher extends CipherSpi { + private static final int ERROR_INVALID_PARAMETER = 0x57; + private static final int NTE_INVALID_PARAMETER = 0x80090027; + // constant for an empty byte array private static final byte[] B0 = new byte[0]; @@ -101,6 +105,8 @@ public final class CRSACipher extends CipherSpi { // cipher parameter for TLS RSA premaster secret private AlgorithmParameterSpec spec = null; + private boolean forTlsPremasterSecret = false; + // the source of randomness private SecureRandom random; @@ -171,6 +177,9 @@ public final class CRSACipher extends CipherSpi { } spec = params; this.random = random; // for TLS RSA premaster secret + this.forTlsPremasterSecret = true; + } else { + this.forTlsPremasterSecret = false; } init(opmode, key); } @@ -278,8 +287,7 @@ public final class CRSACipher extends CipherSpi { } // internal doFinal() method. Here we perform the actual RSA operation - private byte[] doFinal() throws BadPaddingException, - IllegalBlockSizeException { + private byte[] doFinal() throws IllegalBlockSizeException { if (bufOfs > buffer.length) { throw new IllegalBlockSizeException("Data must not be longer " + "than " + (buffer.length - paddingLength) + " bytes"); @@ -308,7 +316,7 @@ public final class CRSACipher extends CipherSpi { throw new AssertionError("Internal error"); } - } catch (KeyException e) { + } catch (KeyException | BadPaddingException e) { throw new ProviderException(e); } finally { @@ -331,14 +339,14 @@ public final class CRSACipher extends CipherSpi { // see JCE spec protected byte[] engineDoFinal(byte[] in, int inOfs, int inLen) - throws BadPaddingException, IllegalBlockSizeException { + throws IllegalBlockSizeException { update(in, inOfs, inLen); return doFinal(); } // see JCE spec protected int engineDoFinal(byte[] in, int inOfs, int inLen, byte[] out, - int outOfs) throws ShortBufferException, BadPaddingException, + int outOfs) throws ShortBufferException, IllegalBlockSizeException { if (outputSize > out.length - outOfs) { throw new ShortBufferException @@ -354,6 +362,7 @@ public final class CRSACipher extends CipherSpi { // see JCE spec protected byte[] engineWrap(Key key) throws InvalidKeyException, IllegalBlockSizeException { + byte[] encoded = key.getEncoded(); // TODO - unextractable key if ((encoded == null) || (encoded.length == 0)) { throw new InvalidKeyException("Could not obtain encoded key"); @@ -362,12 +371,7 @@ public final class CRSACipher extends CipherSpi { throw new InvalidKeyException("Key is too long for wrapping"); } update(encoded, 0, encoded.length); - try { - return doFinal(); - } catch (BadPaddingException e) { - // should not occur - throw new InvalidKeyException("Wrapping failed", e); - } + return doFinal(); } // see JCE spec @@ -388,31 +392,31 @@ public final class CRSACipher extends CipherSpi { update(wrappedKey, 0, wrappedKey.length); try { encoded = doFinal(); - } catch (BadPaddingException e) { - if (isTlsRsaPremasterSecret) { - failover = e; - } else { - throw new InvalidKeyException("Unwrapping failed", e); - } } catch (IllegalBlockSizeException e) { // should not occur, handled with length check above throw new InvalidKeyException("Unwrapping failed", e); } - if (isTlsRsaPremasterSecret) { - if (!(spec instanceof TlsRsaPremasterSecretParameterSpec)) { - throw new IllegalStateException( - "No TlsRsaPremasterSecretParameterSpec specified"); + try { + if (isTlsRsaPremasterSecret) { + if (!forTlsPremasterSecret) { + throw new IllegalStateException( + "No TlsRsaPremasterSecretParameterSpec specified"); + } + + // polish the TLS premaster secret + encoded = KeyUtil.checkTlsPreMasterSecretKey( + ((TlsRsaPremasterSecretParameterSpec) spec).getClientVersion(), + ((TlsRsaPremasterSecretParameterSpec) spec).getServerVersion(), + random, encoded, encoded == null); } - // polish the TLS premaster secret - encoded = KeyUtil.checkTlsPreMasterSecretKey( - ((TlsRsaPremasterSecretParameterSpec)spec).getClientVersion(), - ((TlsRsaPremasterSecretParameterSpec)spec).getServerVersion(), - random, encoded, (failover != null)); + return constructKey(encoded, algorithm, type); + } finally { + if (encoded != null) { + Arrays.fill(encoded, (byte) 0); + } } - - return constructKey(encoded, algorithm, type); } // see JCE spec @@ -496,17 +500,30 @@ public final class CRSACipher extends CipherSpi { * Encrypt/decrypt a data buffer using Microsoft Crypto API or CNG. * It expects and returns ciphertext data in big-endian form. */ - private static byte[] encryptDecrypt(byte[] data, int dataSize, - CKey key, boolean doEncrypt) throws KeyException { + private byte[] encryptDecrypt(byte[] data, int dataSize, + CKey key, boolean doEncrypt) throws KeyException, BadPaddingException { + int[] returnStatus = new int[1]; + byte[] result; if (key.getHCryptKey() != 0) { - return encryptDecrypt(data, dataSize, key.getHCryptKey(), doEncrypt); + result = encryptDecrypt(returnStatus, data, dataSize, key.getHCryptKey(), doEncrypt); } else { - return cngEncryptDecrypt(data, dataSize, key.getHCryptProvider(), doEncrypt); + result = cngEncryptDecrypt(returnStatus, data, dataSize, key.getHCryptProvider(), doEncrypt); } + if ((returnStatus[0] == ERROR_INVALID_PARAMETER) || (returnStatus[0] == NTE_INVALID_PARAMETER)) { + if (forTlsPremasterSecret) { + result = null; + } else { + throw new BadPaddingException("Error " + returnStatus[0] + " returned by MSCAPI"); + } + } else if (returnStatus[0] != 0) { + throw new KeyException("Error " + returnStatus[0] + " returned by MSCAPI"); + } + + return result; } - private static native byte[] encryptDecrypt(byte[] data, int dataSize, + private static native byte[] encryptDecrypt(int[] returnStatus, byte[] data, int dataSize, long key, boolean doEncrypt) throws KeyException; - private static native byte[] cngEncryptDecrypt(byte[] data, int dataSize, + private static native byte[] cngEncryptDecrypt(int[] returnStatus, byte[] data, int dataSize, long key, boolean doEncrypt) throws KeyException; } diff --git a/src/jdk.crypto.mscapi/windows/native/libsunmscapi/security.cpp b/src/jdk.crypto.mscapi/windows/native/libsunmscapi/security.cpp index f2b6bfd83f6..4787708779d 100644 --- a/src/jdk.crypto.mscapi/windows/native/libsunmscapi/security.cpp +++ b/src/jdk.crypto.mscapi/windows/native/libsunmscapi/security.cpp @@ -1905,18 +1905,25 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_CKeyStore_destroyKeyContainer /* * Class: sun_security_mscapi_CRSACipher * Method: encryptDecrypt - * Signature: ([BIJZ)[B + * Signature: ([I[BIJZ)[B */ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_CRSACipher_encryptDecrypt - (JNIEnv *env, jclass clazz, jbyteArray jData, jint jDataSize, jlong hKey, + (JNIEnv *env, jclass clazz, jintArray jResultStatus, jbyteArray jData, jint jDataSize, jlong hKey, jboolean doEncrypt) { jbyteArray result = NULL; jbyte* pData = NULL; + jbyte* resultData = NULL; DWORD dwDataLen = jDataSize; DWORD dwBufLen = env->GetArrayLength(jData); DWORD i; BYTE tmp; + BOOL success; + DWORD ss = ERROR_SUCCESS; + DWORD lastError = ERROR_SUCCESS; + DWORD resultLen = 0; + DWORD pmsLen = 48; + jbyte pmsArr[48] = {0}; __try { @@ -1943,6 +1950,8 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_CRSACipher_encryptDecrypt pData[i] = pData[dwBufLen - i -1]; pData[dwBufLen - i - 1] = tmp; } + resultData = pData; + resultLen = dwBufLen; } else { // convert to little-endian for (i = 0; i < dwBufLen / 2; i++) { @@ -1952,21 +1961,28 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_CRSACipher_encryptDecrypt } // decrypt - if (! ::CryptDecrypt((HCRYPTKEY) hKey, 0, TRUE, 0, (BYTE *)pData, //deprecated - &dwBufLen)) { - - ThrowException(env, KEY_EXCEPTION, GetLastError()); - __leave; + success = ::CryptDecrypt((HCRYPTKEY) hKey, 0, TRUE, 0, (BYTE *)pData, //deprecated + &dwBufLen); + lastError = GetLastError(); + if (success) { + ss = ERROR_SUCCESS; + resultData = pData; + resultLen = dwBufLen; + } else { + ss = lastError; + resultData = pmsArr; + resultLen = pmsLen; } + env->SetIntArrayRegion(jResultStatus, 0, 1, (jint*) &ss); } - // Create new byte array - if ((result = env->NewByteArray(dwBufLen)) == NULL) { + // Create new byte array + if ((result = env->NewByteArray(resultLen)) == NULL) { __leave; } // Copy data from native buffer to Java buffer - env->SetByteArrayRegion(result, 0, dwBufLen, (jbyte*) pData); + env->SetByteArrayRegion(result, 0, resultLen, (jbyte*) resultData); } __finally { @@ -1980,17 +1996,22 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_CRSACipher_encryptDecrypt /* * Class: sun_security_mscapi_CRSACipher * Method: cngEncryptDecrypt - * Signature: ([BIJZ)[B + * Signature: ([I[BIJZ)[B */ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_CRSACipher_cngEncryptDecrypt - (JNIEnv *env, jclass clazz, jbyteArray jData, jint jDataSize, jlong hKey, + (JNIEnv *env, jclass clazz, jintArray jResultStatus, jbyteArray jData, jint jDataSize, jlong hKey, jboolean doEncrypt) { SECURITY_STATUS ss; jbyteArray result = NULL; jbyte* pData = NULL; + jbyte* resultData = NULL; DWORD dwDataLen = jDataSize; DWORD dwBufLen = env->GetArrayLength(jData); + DWORD resultLen = 0; + DWORD pmsLen = 48; + jbyte pmsArr[48] = {0}; + __try { // Copy data from Java buffer to native buffer @@ -2010,6 +2031,9 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_CRSACipher_cngEncryptDecry if (ss != ERROR_SUCCESS) { ThrowException(env, KEY_EXCEPTION, ss); __leave; + } else { + resultLen = dwBufLen; + resultData = pData; } } else { // decrypt @@ -2018,18 +2042,22 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_CRSACipher_cngEncryptDecry 0, (PBYTE)pData, dwBufLen, &dwBufLen, NCRYPT_PAD_PKCS1_FLAG); - if (ss != ERROR_SUCCESS) { - ThrowException(env, KEY_EXCEPTION, ss); - __leave; + env->SetIntArrayRegion(jResultStatus, 0, 1, (jint*) &ss); + if (ss == ERROR_SUCCESS) { + resultLen = dwBufLen; + resultData = pData; + } else { + resultLen = pmsLen; + resultData = pmsArr; } - } + } // Create new byte array - if ((result = env->NewByteArray(dwBufLen)) == NULL) { + if ((result = env->NewByteArray(resultLen)) == NULL) { __leave; } // Copy data from native buffer to Java buffer - env->SetByteArrayRegion(result, 0, dwBufLen, (jbyte*) pData); + env->SetByteArrayRegion(result, 0, resultLen, (jbyte*) resultData); } __finally { if (pData) {