8207317: SSLEngine negotiation fail exception behavior changed from fail-fast to fail-lazy

Reviewed-by: xuelei

src/java.base/share/classes/sun/security/ssl/CertSignAlgsExtension.java

@@ -191,12 +191,12 @@ final class CertSignAlgsExtension {
             }
 
             // update the context
-            List<SignatureScheme> shemes =
+            List<SignatureScheme> schemes =
                     SignatureScheme.getSupportedAlgorithms(
                             shc.algorithmConstraints, shc.negotiatedProtocol,
                             spec.signatureSchemes);
-            shc.peerRequestedCertSignSchemes = shemes;
+            shc.peerRequestedCertSignSchemes = schemes;
-            shc.handshakeSession.setPeerSupportedSignatureAlgorithms(shemes);
+            shc.handshakeSession.setPeerSupportedSignatureAlgorithms(schemes);
 
             if (!shc.isResumption && shc.negotiatedProtocol.useTLS13PlusSpec()) {
                 if (shc.sslConfig.clientAuthType !=
@@ -337,12 +337,12 @@ final class CertSignAlgsExtension {
             }
 
             // update the context
-            List<SignatureScheme> shemes =
+            List<SignatureScheme> schemes =
                     SignatureScheme.getSupportedAlgorithms(
                             chc.algorithmConstraints, chc.negotiatedProtocol,
                             spec.signatureSchemes);
-            chc.peerRequestedCertSignSchemes = shemes;
+            chc.peerRequestedCertSignSchemes = schemes;
-            chc.handshakeSession.setPeerSupportedSignatureAlgorithms(shemes);
+            chc.handshakeSession.setPeerSupportedSignatureAlgorithms(schemes);
         }
     }
 }

src/java.base/share/classes/sun/security/ssl/CertificateMessage.java

@@ -1031,8 +1031,8 @@ final class CertificateMessage {
                 // Don't select a signature scheme unless we will be able to
                 // produce a CertificateVerify message later
                 if (SignatureScheme.getPreferableAlgorithm(
-                    hc.peerRequestedSignatureSchemes,
+                        hc.peerRequestedSignatureSchemes,
-                    ss, hc.negotiatedProtocol) == null) {
+                        ss, hc.negotiatedProtocol) == null) {
 
                     if (SSLLogger.isOn && SSLLogger.isOn("ssl,handshake")) {
                         SSLLogger.warning(

src/java.base/share/classes/sun/security/ssl/HandshakeOutStream.java

@@ -50,7 +50,7 @@ public class HandshakeOutStream extends ByteArrayOutputStream {
         this.outputRecord = outputRecord;
     }
 
-    // Complete a handshakin message writing. Called by HandshakeMessage.
+    // Complete a handshaking message write. Called by HandshakeMessage.
     void complete() throws IOException {
         if (size() < 4) {       // 4: handshake message header size
             // internal_error alert will be triggered

src/java.base/share/classes/sun/security/ssl/SSLCipher.java

@@ -379,10 +379,10 @@ enum SSLCipher {
     private final Map.Entry<WriteCipherGenerator,
             ProtocolVersion[]>[] writeCipherGenerators;
 
-    // Map of Ciphers listed in jdk.tls.KeyLimit
+    // Map of Ciphers listed in jdk.tls.keyLimits
     private static final HashMap<String, Long> cipherLimits = new HashMap<>();
 
-    // Keywords found on the jdk.tls.KeyLimit security property.
+    // Keywords found on the jdk.tls.keyLimits security property.
     final static String tag[] = {"KEYUPDATE"};
 
     static  {
@@ -407,7 +407,7 @@ enum SSLCipher {
                     index = 0;
                 } else {
                     if (SSLLogger.isOn && SSLLogger.isOn("ssl")) {
-                        SSLLogger.fine("jdk.net.keyLimits:  Unknown action:  " +
+                        SSLLogger.fine("jdk.tls.keyLimits:  Unknown action:  " +
                                 entry);
                     }
                     continue;
@@ -423,17 +423,18 @@ enum SSLCipher {
                         size = Long.parseLong(values[2]);
                     }
                     if (size < 1 || size > max) {
-                        throw new NumberFormatException("Length exceeded limits");
+                        throw new NumberFormatException(
+                            "Length exceeded limits");
                     }
                 } catch (NumberFormatException e) {
                     if (SSLLogger.isOn && SSLLogger.isOn("ssl")) {
-                        SSLLogger.fine("jdk.net.keyLimits:  " + e.getMessage() +
+                        SSLLogger.fine("jdk.tls.keyLimits:  " + e.getMessage() +
                                 ":  " +  entry);
                     }
                     continue;
                 }
                 if (SSLLogger.isOn && SSLLogger.isOn("ssl")) {
-                    SSLLogger.fine("jdk.net.keyLimits:  entry = " + entry +
+                    SSLLogger.fine("jdk.tls.keyLimits:  entry = " + entry +
                             ". " + values[0] + ":" + tag[index] + " = " + size);
                 }
                 cipherLimits.put(values[0] + ":" + tag[index], size);

src/java.base/share/classes/sun/security/ssl/SSLEngineImpl.java

@@ -127,9 +127,7 @@ final class SSLEngineImpl extends SSLEngine implements SSLTransport {
         }
 
         // See if the handshaker needs to report back some SSLException.
-        if (conContext.outputRecord.isEmpty()) {
+        checkTaskThrown();
-            checkTaskThrown();
-        }   // Otherwise, deliver cached records before throwing task exception.
 
         // check parameters
         checkParams(srcs, srcsOffset, srcsLength, dsts, dstsOffset, dstsLength);
@@ -896,18 +894,58 @@ final class SSLEngineImpl extends SSLEngine implements SSLTransport {
         return true;
     }
 
+    /*
+     * Depending on whether the error was just a warning and the
+     * handshaker wasn't closed, or fatal and the handshaker is now
+     * null, report back the Exception that happened in the delegated
+     * task(s).
+     */
     private synchronized void checkTaskThrown() throws SSLException {
+
+        Exception exc = null;
+
+        // First check the handshake context.
         HandshakeContext hc = conContext.handshakeContext;
-        if (hc != null && hc.delegatedThrown != null) {
+        if ((hc != null) && (hc.delegatedThrown != null)) {
-            try {
+            exc = hc.delegatedThrown;
-                throw getTaskThrown(hc.delegatedThrown);
+            hc.delegatedThrown = null;
-            } finally {
+        }
-                hc.delegatedThrown = null;
+
+        /*
+         * hc.delegatedThrown and conContext.delegatedThrown are most likely
+         * the same, but it's possible we could have had a non-fatal
+         * exception and thus the new HandshakeContext is still valid
+         * (alert warning).  If so, then we may have a secondary exception
+         * waiting to be reported from the TransportContext, so we will
+         * need to clear that on a successive call.  Otherwise, clear it now.
+         */
+        if (conContext.delegatedThrown != null) {
+            if (exc != null) {
+                // hc object comparison
+                if (conContext.delegatedThrown == exc) {
+                    // clear if/only if both are the same
+                    conContext.delegatedThrown = null;
+                } // otherwise report the hc delegatedThrown
+            } else {
+                // Nothing waiting in HandshakeContext, but one is in the
+                // TransportContext.
+                exc = conContext.delegatedThrown;
+                conContext.delegatedThrown = null;
             }
         }
 
-        if (conContext.isBroken && conContext.closeReason != null) {
+        // Anything to report?
-            throw getTaskThrown(conContext.closeReason);
+        if (exc == null) {
+            return;
+        }
+
+        // If it wasn't a RuntimeException/SSLException, need to wrap it.
+        if (exc instanceof SSLException) {
+            throw (SSLException)exc;
+        } else if (exc instanceof RuntimeException) {
+            throw (RuntimeException)exc;
+        } else {
+            throw getTaskThrown(exc);
         }
     }
 
@@ -963,20 +1001,41 @@ final class SSLEngineImpl extends SSLEngine implements SSLTransport {
                 } catch (PrivilegedActionException pae) {
                     // Get the handshake context again in case the
                     // handshaking has completed.
+                    Exception reportedException = pae.getException();
+
+                    // Report to both the TransportContext...
+                    if (engine.conContext.delegatedThrown == null) {
+                        engine.conContext.delegatedThrown = reportedException;
+                    }
+
+                    // ...and the HandshakeContext in case condition
+                    // wasn't fatal and the handshakeContext is still
+                    // around.
                     hc = engine.conContext.handshakeContext;
                     if (hc != null) {
-                        hc.delegatedThrown = pae.getException();
+                        hc.delegatedThrown = reportedException;
                     } else if (engine.conContext.closeReason != null) {
+                        // Update the reason in case there was a previous.
                         engine.conContext.closeReason =
-                                getTaskThrown(pae.getException());
+                                getTaskThrown(reportedException);
                     }
                 } catch (RuntimeException rte) {
                     // Get the handshake context again in case the
                     // handshaking has completed.
+
+                    // Report to both the TransportContext...
+                    if (engine.conContext.delegatedThrown == null) {
+                        engine.conContext.delegatedThrown = rte;
+                    }
+
+                    // ...and the HandshakeContext in case condition
+                    // wasn't fatal and the handshakeContext is still
+                    // around.
                     hc = engine.conContext.handshakeContext;
                     if (hc != null) {
                         hc.delegatedThrown = rte;
                     } else if (engine.conContext.closeReason != null) {
+                        // Update the reason in case there was a previous.
                         engine.conContext.closeReason = rte;
                     }
                 }
@@ -1000,13 +1059,6 @@ final class SSLEngineImpl extends SSLEngine implements SSLTransport {
             @Override
             public Void run() throws Exception {
                 while (!context.delegatedActions.isEmpty()) {
-                    // Report back the task SSLException
-                    if (context.delegatedThrown != null) {
-                        Exception delegatedThrown = context.delegatedThrown;
-                        context.delegatedThrown = null;
-                        throw getTaskThrown(delegatedThrown);
-                    }
-
                     Map.Entry<Byte, ByteBuffer> me =
                             context.delegatedActions.poll();
                     if (me != null) {

src/java.base/share/classes/sun/security/ssl/SSLTransport.java

@@ -167,9 +167,10 @@ interface SSLTransport {
             if (plainText == null) {
                 plainText = Plaintext.PLAINTEXT_NULL;
             } else {
-                // File the destination buffers.
+                // Fill the destination buffers.
-                if (dsts != null && dstsLength > 0 &&
+                if ((dsts != null) && (dstsLength > 0) &&
-                    plainText.contentType == ContentType.APPLICATION_DATA.id) {
+                        (plainText.contentType ==
+                            ContentType.APPLICATION_DATA.id)) {
 
                     ByteBuffer fragment = plainText.fragment;
                     int remains = fragment.remaining();

src/java.base/share/classes/sun/security/ssl/ServerHello.java

@@ -40,6 +40,7 @@ import javax.crypto.SecretKey;
 import javax.crypto.spec.IvParameterSpec;
 import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLProtocolException;
 import sun.security.ssl.CipherSuite.KeyExchange;
 import sun.security.ssl.ClientHello.ClientHelloMessage;
 import sun.security.ssl.SSLCipher.SSLReadCipher;
@@ -139,8 +140,11 @@ final class ServerHello {
 
             this.serverRandom = new RandomCookie(m);
             this.sessionId = new SessionId(Record.getBytes8(m));
-            sessionId.checkLength(serverVersion.id);
+            try {
-
+                sessionId.checkLength(serverVersion.id);
+            } catch (SSLProtocolException ex) {
+                handshakeContext.conContext.fatal(Alert.ILLEGAL_PARAMETER, ex);
+            }
 
             int cipherSuiteId = Record.getInt16(m);
             this.cipherSuite = CipherSuite.valueOf(cipherSuiteId);

src/java.base/share/classes/sun/security/ssl/SignatureAlgorithmsExtension.java

@@ -362,16 +362,16 @@ final class SignatureAlgorithmsExtension {
                 // certificates and server key exchange), it MUST send the
                 // signature_algorithms extension, listing the algorithms it
                 // is willing to accept.
-                List<SignatureScheme> shemes = Arrays.asList(
+                List<SignatureScheme> schemes = Arrays.asList(
                         SignatureScheme.RSA_PKCS1_SHA1,
                         SignatureScheme.DSA_SHA1,
                         SignatureScheme.ECDSA_SHA1
                 );
 
-                shc.peerRequestedSignatureSchemes = shemes;
+                shc.peerRequestedSignatureSchemes = schemes;
                 if (shc.peerRequestedCertSignSchemes == null ||
-                    shc.peerRequestedCertSignSchemes.isEmpty()) {
+                        shc.peerRequestedCertSignSchemes.isEmpty()) {
-                        shc.peerRequestedCertSignSchemes = shemes;
+                    shc.peerRequestedCertSignSchemes = schemes;
                 }
 
                 // Use the default peer signature algorithms.

src/java.base/share/classes/sun/security/ssl/SignatureScheme.java

@@ -403,8 +403,8 @@ enum SignatureScheme {
 
         for (SignatureScheme ss : schemes) {
             if (ss.isAvailable &&
-            ss.handshakeSupportedProtocols.contains(version) &&
+                    ss.handshakeSupportedProtocols.contains(version) &&
-            certScheme.keyAlgorithm.equalsIgnoreCase(ss.keyAlgorithm)) {
+                    certScheme.keyAlgorithm.equalsIgnoreCase(ss.keyAlgorithm)) {
 
                 return ss;
             }

src/java.base/share/classes/sun/security/ssl/TransportContext.java

@@ -63,6 +63,7 @@ class TransportContext implements ConnectionContext {
     boolean                         isInputCloseNotified = false;
     boolean                         peerUserCanceled = false;
     Exception                       closeReason = null;
+    Exception                       delegatedThrown = null;
 
     // negotiated security parameters
     SSLSessionImpl                  conSession;
@@ -364,12 +365,12 @@ class TransportContext implements ConnectionContext {
             }
         }
 
-        // terminal handshake context
+        // terminate the handshake context
         if (handshakeContext != null) {
             handshakeContext = null;
         }
 
-        // terminal the transport
+        // terminate the transport
         try {
             transport.shutdown();
         } catch (IOException ioe) {

src/java.base/share/classes/sun/security/ssl/X509Authentication.java

@@ -73,7 +73,7 @@ enum X509Authentication implements SSLAuthentication {
     }
 
     static X509Authentication valueOf(SignatureScheme signatureScheme) {
-        for (X509Authentication au: X509Authentication.values()) {
+        for (X509Authentication au : X509Authentication.values()) {
             if (au.keyType.equals(signatureScheme.keyAlgorithm)) {
                 return au;
             }
@@ -291,9 +291,9 @@ enum X509Authentication implements SSLAuthentication {
                         ((ECPublicKey)serverPublicKey).getParams();
                 NamedGroup namedGroup = NamedGroup.valueOf(params);
                 if ((namedGroup == null) ||
-                    (!SupportedGroups.isSupported(namedGroup)) ||
+                        (!SupportedGroups.isSupported(namedGroup)) ||
-                    ((shc.clientRequestedNamedGroups != null) &&
+                        ((shc.clientRequestedNamedGroups != null) &&
-                    !shc.clientRequestedNamedGroups.contains(namedGroup))) {
+                        !shc.clientRequestedNamedGroups.contains(namedGroup))) {
 
                     if (SSLLogger.isOn && SSLLogger.isOn("ssl")) {
                         SSLLogger.warning(

test/jdk/javax/net/ssl/templates/SSLEngineTemplate.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,7 +30,6 @@
  * @summary SSLEngine has not yet caused Solaris kernel to panic
  * @run main/othervm SSLEngineTemplate
  */
-
 /**
  * A SSLEngine usage example which simplifies the presentation
  * by removing the I/O and multi-threading concerns.
@@ -66,7 +65,6 @@
  *      unwrap()        ...             ChangeCipherSpec
  *      unwrap()        ...             Finished
  */
-
 import javax.net.ssl.*;
 import javax.net.ssl.SSLEngineResult.*;
 import java.io.*;
@@ -115,7 +113,7 @@ public class SSLEngineTemplate {
     private static final String pathToStores = "../etc";
     private static final String keyStoreFile = "keystore";
     private static final String trustStoreFile = "truststore";
-    private static final String passwd = "passphrase";
+    private static final char[] passphrase = "passphrase".toCharArray();
 
     private static final String keyFilename =
             System.getProperty("test.src", ".") + "/" + pathToStores +
@@ -146,8 +144,6 @@ public class SSLEngineTemplate {
         KeyStore ks = KeyStore.getInstance("JKS");
         KeyStore ts = KeyStore.getInstance("JKS");
 
-        char[] passphrase = "passphrase".toCharArray();
-
         ks.load(new FileInputStream(keyFilename), passphrase);
         ts.load(new FileInputStream(trustFilename), passphrase);
 
@@ -187,8 +183,11 @@ public class SSLEngineTemplate {
         createSSLEngines();
         createBuffers();
 
-        SSLEngineResult clientResult;   // results from client's last operation
+        // results from client's last operation
-        SSLEngineResult serverResult;   // results from server's last operation
+        SSLEngineResult clientResult;
+
+        // results from server's last operation
+        SSLEngineResult serverResult;
 
         /*
          * Examining the SSLEngineResults could be much more involved,
@@ -198,31 +197,62 @@ public class SSLEngineTemplate {
          * to write to the output pipe, we could reallocate a larger
          * pipe, but instead we wait for the peer to drain it.
          */
-        while (!isEngineClosed(clientEngine) ||
+        Exception clientException = null;
-                !isEngineClosed(serverEngine)) {
+        Exception serverException = null;
+
+        while (!isEngineClosed(clientEngine)
+                || !isEngineClosed(serverEngine)) {
 
             log("================");
 
-            clientResult = clientEngine.wrap(clientOut, cTOs);
+            try {
-            log("client wrap: ", clientResult);
+                clientResult = clientEngine.wrap(clientOut, cTOs);
-            runDelegatedTasks(clientResult, clientEngine);
+                log("client wrap: ", clientResult);
+            } catch (Exception e) {
+                clientException = e;
+                System.out.println("Client wrap() threw: " + e.getMessage());
+            }
+            logEngineStatus(clientEngine);
+            runDelegatedTasks(clientEngine);
 
-            serverResult = serverEngine.wrap(serverOut, sTOc);
+            log("----");
-            log("server wrap: ", serverResult);
+
-            runDelegatedTasks(serverResult, serverEngine);
+            try {
+                serverResult = serverEngine.wrap(serverOut, sTOc);
+                log("server wrap: ", serverResult);
+            } catch (Exception e) {
+                serverException = e;
+                System.out.println("Server wrap() threw: " + e.getMessage());
+            }
+            logEngineStatus(serverEngine);
+            runDelegatedTasks(serverEngine);
 
             cTOs.flip();
             sTOc.flip();
 
+            log("--------");
+
+            try {
+                clientResult = clientEngine.unwrap(sTOc, clientIn);
+                log("client unwrap: ", clientResult);
+            } catch (Exception e) {
+                clientException = e;
+                System.out.println("Client unwrap() threw: " + e.getMessage());
+            }
+            logEngineStatus(clientEngine);
+            runDelegatedTasks(clientEngine);
+
             log("----");
 
-            clientResult = clientEngine.unwrap(sTOc, clientIn);
+            try {
-            log("client unwrap: ", clientResult);
+                serverResult = serverEngine.unwrap(cTOs, serverIn);
-            runDelegatedTasks(clientResult, clientEngine);
+                log("server unwrap: ", serverResult);
-
+            } catch (Exception e) {
-            serverResult = serverEngine.unwrap(cTOs, serverIn);
+                serverException = e;
-            log("server unwrap: ", serverResult);
+                System.out.println("Server unwrap() threw: " + e.getMessage());
-            runDelegatedTasks(serverResult, serverEngine);
+            }
+            logEngineStatus(serverEngine);
+            runDelegatedTasks(serverEngine);
 
             cTOs.compact();
             sTOc.compact();
@@ -244,13 +274,22 @@ public class SSLEngineTemplate {
 
                 log("\tClosing clientEngine's *OUTBOUND*...");
                 clientEngine.closeOutbound();
+                logEngineStatus(clientEngine);
+
                 dataDone = true;
                 log("\tClosing serverEngine's *OUTBOUND*...");
                 serverEngine.closeOutbound();
+                logEngineStatus(serverEngine);
             }
         }
     }
 
+    private static void logEngineStatus(SSLEngine engine) {
+        log("\tCurrent HS State  " + engine.getHandshakeStatus().toString());
+        log("\tisInboundDone():  " + engine.isInboundDone());
+        log("\tisOutboundDone(): " + engine.isOutboundDone());
+    }
+
     /*
      * Using the SSLContext created during object creation,
      * create/configure the SSLEngines we'll use for this test.
@@ -264,11 +303,19 @@ public class SSLEngineTemplate {
         serverEngine.setUseClientMode(false);
         serverEngine.setNeedClientAuth(true);
 
+        // Get/set parameters if needed
+        SSLParameters paramsServer = serverEngine.getSSLParameters();
+        serverEngine.setSSLParameters(paramsServer);
+
         /*
          * Similar to above, but using client mode instead.
          */
         clientEngine = sslc.createSSLEngine("client", 80);
         clientEngine.setUseClientMode(true);
+
+        // Get/set parameters if needed
+        SSLParameters paramsClient = clientEngine.getSSLParameters();
+        clientEngine.setSSLParameters(paramsClient);
     }
 
     /*
@@ -307,13 +354,12 @@ public class SSLEngineTemplate {
      * If the result indicates that we have outstanding tasks to do,
      * go ahead and run them in this thread.
      */
-    private static void runDelegatedTasks(SSLEngineResult result,
+    private static void runDelegatedTasks(SSLEngine engine) throws Exception {
-            SSLEngine engine) throws Exception {
 
-        if (result.getHandshakeStatus() == HandshakeStatus.NEED_TASK) {
+        if (engine.getHandshakeStatus() == HandshakeStatus.NEED_TASK) {
             Runnable runnable;
             while ((runnable = engine.getDelegatedTask()) != null) {
-                log("\trunning delegated task...");
+                log("    running delegated task...");
                 runnable.run();
             }
             HandshakeStatus hsStatus = engine.getHandshakeStatus();
@@ -321,7 +367,7 @@ public class SSLEngineTemplate {
                 throw new Exception(
                     "handshake shouldn't need additional tasks");
             }
-            log("\tnew HandshakeStatus: " + hsStatus);
+            logEngineStatus(engine);
         }
     }
 
@@ -361,14 +407,14 @@ public class SSLEngineTemplate {
         if (resultOnce) {
             resultOnce = false;
             System.out.println("The format of the SSLEngineResult is: \n" +
-                "\t\"getStatus() / getHandshakeStatus()\" +\n" +
+                    "\t\"getStatus() / getHandshakeStatus()\" +\n" +
-                "\t\"bytesConsumed() / bytesProduced()\"\n");
+                    "\t\"bytesConsumed() / bytesProduced()\"\n");
         }
         HandshakeStatus hsStatus = result.getHandshakeStatus();
         log(str +
-            result.getStatus() + "/" + hsStatus + ", " +
+                result.getStatus() + "/" + hsStatus + ", " +
-            result.bytesConsumed() + "/" + result.bytesProduced() +
+                result.bytesConsumed() + "/" + result.bytesProduced() +
-            " bytes");
+                " bytes");
         if (hsStatus == HandshakeStatus.FINISHED) {
             log("\t...ready for application data");
         }

test/jdk/sun/security/ssl/ClientHandshaker/LengthCheckTest.java

@@ -74,6 +74,7 @@ import java.security.*;
 import java.nio.*;
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Iterator;
 
 public class LengthCheckTest {
 
@@ -203,7 +204,11 @@ public class LengthCheckTest {
 
                 // Now send each ByteBuffer (each being a complete
                 // TLS record) into the client-side unwrap.
-                for (ByteBuffer bBuf : recList) {
+                // for (ByteBuffer bBuf : recList) {
+
+                Iterator<ByteBuffer> iter = recList.iterator();
+                while (!gotException && (iter.hasNext())) {
+                    ByteBuffer bBuf = iter.next();
                     dumpByteBuffer("SERVER-TO-CLIENT", bBuf);
                     try {
                         clientResult = clientEngine.unwrap(bBuf, clientIn);
@@ -232,8 +237,8 @@ public class LengthCheckTest {
             // was thrown and the proper action (a TLS alert) was
             // sent back to the server.
             if (gotException == false ||
-                !isTlsMessage(cTOs, TLS_RECTYPE_ALERT, TLS_ALERT_LVL_FATAL,
+                    !isTlsMessage(cTOs, TLS_RECTYPE_ALERT, TLS_ALERT_LVL_FATAL,
-                        TLS_ALERT_UNEXPECTED_MSG)) {
+                            TLS_ALERT_ILLEGAL_PARAMETER)) {
                 throw new SSLException(
                     "Client failed to throw Alert:fatal:internal_error");
             }
@@ -253,38 +258,36 @@ public class LengthCheckTest {
             ByteBuffer evilClientHello = createEvilClientHello(64);
             dumpByteBuffer("CLIENT-TO-SERVER", evilClientHello);
 
+            // Server consumes Client Hello
+            serverResult = serverEngine.unwrap(evilClientHello, serverIn);
+            log("server unwrap: ", serverResult);
+            runDelegatedTasks(serverResult, serverEngine);
+            evilClientHello.compact();
+
+            // Under normal circumstances this should be a ServerHello
+            // But should throw an exception instead due to the invalid
+            // session ID.
             try {
-                // Server consumes Client Hello
-                serverResult = serverEngine.unwrap(evilClientHello, serverIn);
-                log("server unwrap: ", serverResult);
-                runDelegatedTasks(serverResult, serverEngine);
-                evilClientHello.compact();
-
-                // Under normal circumstances this should be a ServerHello
-                // But should throw an exception instead due to the invalid
-                // session ID.
                 serverResult = serverEngine.wrap(serverOut, sTOc);
                 log("server wrap: ", serverResult);
                 runDelegatedTasks(serverResult, serverEngine);
-                sTOc.flip();
-                dumpByteBuffer("SERVER-TO-CLIENT", sTOc);
-
-                // We expect to see the server generate an alert here
-                serverResult = serverEngine.wrap(serverOut, sTOc);
-                log("server wrap: ", serverResult);
-                runDelegatedTasks(serverResult, serverEngine);
-                sTOc.flip();
-                dumpByteBuffer("SERVER-TO-CLIENT", sTOc);
             } catch (SSLProtocolException ssle) {
                 log("Received expected SSLProtocolException: " + ssle);
                 gotException = true;
             }
 
+            // We expect to see the server generate an alert here
+            serverResult = serverEngine.wrap(serverOut, sTOc);
+            log("server wrap: ", serverResult);
+            runDelegatedTasks(serverResult, serverEngine);
+            sTOc.flip();
+            dumpByteBuffer("SERVER-TO-CLIENT", sTOc);
+
             // At this point we can verify that both an exception
             // was thrown and the proper action (a TLS alert) was
             // sent back to the client.
             if (gotException == false ||
-                !isTlsMessage(sTOc, TLS_RECTYPE_ALERT, TLS_ALERT_LVL_FATAL,
+                    !isTlsMessage(sTOc, TLS_RECTYPE_ALERT, TLS_ALERT_LVL_FATAL,
                         TLS_ALERT_ILLEGAL_PARAMETER)) {
                 throw new SSLException(
                     "Server failed to throw Alert:fatal:internal_error");

test/jdk/sun/security/ssl/SSLEngineImpl/SSLEngineFailedALPN.java

@@ -0,0 +1,449 @@
+/*
+ * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+// SunJSSE does not support dynamic system properties, no way to re-use
+// system properties in samevm/agentvm mode.
+
+/*
+ * @test
+ * @bug 8207317
+ * @summary SSLEngine negotiation fail Exception behavior changed from
+ *          fail-fast to fail-lazy
+ * @run main/othervm SSLEngineFailedALPN
+ */
+/**
+ * A SSLEngine usage example which simplifies the presentation
+ * by removing the I/O and multi-threading concerns.
+ *
+ * The test creates two SSLEngines, simulating a client and server.
+ * The "transport" layer consists two byte buffers:  think of them
+ * as directly connected pipes.
+ *
+ * Note, this is a *very* simple example: real code will be much more
+ * involved.  For example, different threading and I/O models could be
+ * used, transport mechanisms could close unexpectedly, and so on.
+ *
+ * When this application runs, notice that several messages
+ * (wrap/unwrap) pass before any application data is consumed or
+ * produced.  (For more information, please see the SSL/TLS
+ * specifications.)  There may several steps for a successful handshake,
+ * so it's typical to see the following series of operations:
+ *
+ *      client          server          message
+ *      ======          ======          =======
+ *      wrap()          ...             ClientHello
+ *      ...             unwrap()        ClientHello
+ *      ...             wrap()          ServerHello/Certificate
+ *      unwrap()        ...             ServerHello/Certificate
+ *      wrap()          ...             ClientKeyExchange
+ *      wrap()          ...             ChangeCipherSpec
+ *      wrap()          ...             Finished
+ *      ...             unwrap()        ClientKeyExchange
+ *      ...             unwrap()        ChangeCipherSpec
+ *      ...             unwrap()        Finished
+ *      ...             wrap()          ChangeCipherSpec
+ *      ...             wrap()          Finished
+ *      unwrap()        ...             ChangeCipherSpec
+ *      unwrap()        ...             Finished
+ */
+import javax.net.ssl.*;
+import javax.net.ssl.SSLEngineResult.*;
+import java.io.*;
+import java.security.*;
+import java.nio.*;
+
+public class SSLEngineFailedALPN {
+
+    /*
+     * Enables logging of the SSLEngine operations.
+     */
+    private static final boolean logging = true;
+
+    /*
+     * Enables the JSSE system debugging system property:
+     *
+     *     -Djavax.net.debug=all
+     *
+     * This gives a lot of low-level information about operations underway,
+     * including specific handshake messages, and might be best examined
+     * after gaining some familiarity with this application.
+     */
+    private static final boolean debug = false;
+
+    private final SSLContext sslc;
+
+    private SSLEngine clientEngine;     // client Engine
+    private ByteBuffer clientOut;       // write side of clientEngine
+    private ByteBuffer clientIn;        // read side of clientEngine
+
+    private SSLEngine serverEngine;     // server Engine
+    private ByteBuffer serverOut;       // write side of serverEngine
+    private ByteBuffer serverIn;        // read side of serverEngine
+
+    /*
+     * For data transport, this example uses local ByteBuffers.  This
+     * isn't really useful, but the purpose of this example is to show
+     * SSLEngine concepts, not how to do network transport.
+     */
+    private ByteBuffer cTOs;            // "reliable" transport client->server
+    private ByteBuffer sTOc;            // "reliable" transport server->client
+
+    /*
+     * The following is to set up the keystores.
+     */
+    private static final String pathToStores = "../../../../javax/net/ssl/etc";
+    private static final String keyStoreFile = "keystore";
+    private static final String trustStoreFile = "truststore";
+    private static final char[] passphrase = "passphrase".toCharArray();
+
+    private static final String keyFilename =
+            System.getProperty("test.src", ".") + "/" + pathToStores +
+                "/" + keyStoreFile;
+    private static final String trustFilename =
+            System.getProperty("test.src", ".") + "/" + pathToStores +
+                "/" + trustStoreFile;
+
+    /*
+     * Main entry point for this test.
+     */
+    public static void main(String args[]) throws Exception {
+        if (debug) {
+            System.setProperty("javax.net.debug", "all");
+        }
+
+        SSLEngineFailedALPN test = new SSLEngineFailedALPN();
+        test.runTest();
+
+        System.out.println("Test Passed.");
+    }
+
+    /*
+     * Create an initialized SSLContext to use for these tests.
+     */
+    public SSLEngineFailedALPN() throws Exception {
+
+        KeyStore ks = KeyStore.getInstance("JKS");
+        KeyStore ts = KeyStore.getInstance("JKS");
+
+        ks.load(new FileInputStream(keyFilename), passphrase);
+        ts.load(new FileInputStream(trustFilename), passphrase);
+
+        KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
+        kmf.init(ks, passphrase);
+
+        TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
+        tmf.init(ts);
+
+        SSLContext sslCtx = SSLContext.getInstance("TLS");
+
+        sslCtx.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
+
+        sslc = sslCtx;
+    }
+
+    /*
+     * Run the test.
+     *
+     * Sit in a tight loop, both engines calling wrap/unwrap regardless
+     * of whether data is available or not.  We do this until both engines
+     * report back they are closed.
+     *
+     * The main loop handles all of the I/O phases of the SSLEngine's
+     * lifetime:
+     *
+     *     initial handshaking
+     *     application data transfer
+     *     engine closing
+     *
+     * One could easily separate these phases into separate
+     * sections of code.
+     */
+    private void runTest() throws Exception {
+        boolean dataDone = false;
+
+        createSSLEngines();
+        createBuffers();
+
+        // results from client's last operation
+        SSLEngineResult clientResult;
+
+        // results from server's last operation
+        SSLEngineResult serverResult;
+
+        /*
+         * Examining the SSLEngineResults could be much more involved,
+         * and may alter the overall flow of the application.
+         *
+         * For example, if we received a BUFFER_OVERFLOW when trying
+         * to write to the output pipe, we could reallocate a larger
+         * pipe, but instead we wait for the peer to drain it.
+         */
+        Exception clientException = null;
+        Exception serverException = null;
+
+        while (!isEngineClosed(clientEngine)
+                || !isEngineClosed(serverEngine)) {
+
+            log("================");
+
+            try {
+                clientResult = clientEngine.wrap(clientOut, cTOs);
+                log("client wrap: ", clientResult);
+            } catch (Exception e) {
+                clientException = e;
+                System.out.println("Client wrap() threw: " + e.getMessage());
+            }
+            logEngineStatus(clientEngine);
+            runDelegatedTasks(clientEngine);
+
+            log("----");
+
+            try {
+                serverResult = serverEngine.wrap(serverOut, sTOc);
+                log("server wrap: ", serverResult);
+            } catch (Exception e) {
+                serverException = e;
+                System.out.println("Server wrap() threw: " + e.getMessage());
+            }
+            logEngineStatus(serverEngine);
+            runDelegatedTasks(serverEngine);
+
+            cTOs.flip();
+            sTOc.flip();
+
+            log("--------");
+
+            try {
+                clientResult = clientEngine.unwrap(sTOc, clientIn);
+                log("client unwrap: ", clientResult);
+            } catch (Exception e) {
+                clientException = e;
+                System.out.println("Client unwrap() threw: " + e.getMessage());
+            }
+            logEngineStatus(clientEngine);
+            runDelegatedTasks(clientEngine);
+
+            log("----");
+
+            try {
+                serverResult = serverEngine.unwrap(cTOs, serverIn);
+                log("server unwrap: ", serverResult);
+            } catch (Exception e) {
+                serverException = e;
+                System.out.println("Server unwrap() threw: " + e.getMessage());
+            }
+            logEngineStatus(serverEngine);
+            runDelegatedTasks(serverEngine);
+
+            cTOs.compact();
+            sTOc.compact();
+
+            /*
+             * After we've transfered all application data between the client
+             * and server, we close the clientEngine's outbound stream.
+             * This generates a close_notify handshake message, which the
+             * server engine receives and responds by closing itself.
+             */
+            if (!dataDone && (clientOut.limit() == serverIn.position()) &&
+                    (serverOut.limit() == clientIn.position())) {
+
+                /*
+                 * A sanity check to ensure we got what was sent.
+                 */
+                checkTransfer(serverOut, clientIn);
+                checkTransfer(clientOut, serverIn);
+
+                log("\tClosing clientEngine's *OUTBOUND*...");
+                clientEngine.closeOutbound();
+                logEngineStatus(clientEngine);
+
+                dataDone = true;
+                log("\tClosing serverEngine's *OUTBOUND*...");
+                serverEngine.closeOutbound();
+                logEngineStatus(serverEngine);
+            }
+        }
+
+        log("================");
+
+        if ((clientException != null) &&
+                (clientException instanceof SSLHandshakeException)) {
+            log("Client threw proper exception");
+            clientException.printStackTrace(System.out);
+        } else {
+            throw new Exception("Client Exception not seen");
+        }
+
+        if ((serverException != null) &&
+                (serverException instanceof SSLHandshakeException)) {
+            log("Server threw proper exception:");
+            serverException.printStackTrace(System.out);
+        } else {
+            throw new Exception("Server Exception not seen");
+        }
+    }
+
+    private static void logEngineStatus(SSLEngine engine) {
+        log("\tCurrent HS State  " + engine.getHandshakeStatus().toString());
+        log("\tisInboundDone():  " + engine.isInboundDone());
+        log("\tisOutboundDone(): " + engine.isOutboundDone());
+    }
+
+    /*
+     * Using the SSLContext created during object creation,
+     * create/configure the SSLEngines we'll use for this test.
+     */
+    private void createSSLEngines() throws Exception {
+        /*
+         * Configure the serverEngine to act as a server in the SSL/TLS
+         * handshake.  Also, require SSL client authentication.
+         */
+        serverEngine = sslc.createSSLEngine();
+        serverEngine.setUseClientMode(false);
+        serverEngine.setNeedClientAuth(true);
+
+        // Get/set parameters if needed
+        SSLParameters paramsServer = serverEngine.getSSLParameters();
+        paramsServer.setApplicationProtocols(new String[]{"one"});
+        serverEngine.setSSLParameters(paramsServer);
+
+        /*
+         * Similar to above, but using client mode instead.
+         */
+        clientEngine = sslc.createSSLEngine("client", 80);
+        clientEngine.setUseClientMode(true);
+
+        // Get/set parameters if needed
+        SSLParameters paramsClient = clientEngine.getSSLParameters();
+        paramsClient.setApplicationProtocols(new String[]{"two"});
+        clientEngine.setSSLParameters(paramsClient);
+    }
+
+    /*
+     * Create and size the buffers appropriately.
+     */
+    private void createBuffers() {
+
+        /*
+         * We'll assume the buffer sizes are the same
+         * between client and server.
+         */
+        SSLSession session = clientEngine.getSession();
+        int appBufferMax = session.getApplicationBufferSize();
+        int netBufferMax = session.getPacketBufferSize();
+
+        /*
+         * We'll make the input buffers a bit bigger than the max needed
+         * size, so that unwrap()s following a successful data transfer
+         * won't generate BUFFER_OVERFLOWS.
+         *
+         * We'll use a mix of direct and indirect ByteBuffers for
+         * tutorial purposes only.  In reality, only use direct
+         * ByteBuffers when they give a clear performance enhancement.
+         */
+        clientIn = ByteBuffer.allocate(appBufferMax + 50);
+        serverIn = ByteBuffer.allocate(appBufferMax + 50);
+
+        cTOs = ByteBuffer.allocateDirect(netBufferMax);
+        sTOc = ByteBuffer.allocateDirect(netBufferMax);
+
+        clientOut = ByteBuffer.wrap("Hi Server, I'm Client".getBytes());
+        serverOut = ByteBuffer.wrap("Hello Client, I'm Server".getBytes());
+    }
+
+    /*
+     * If the result indicates that we have outstanding tasks to do,
+     * go ahead and run them in this thread.
+     */
+    private static void runDelegatedTasks(SSLEngine engine) throws Exception {
+
+        if (engine.getHandshakeStatus() == HandshakeStatus.NEED_TASK) {
+            Runnable runnable;
+            while ((runnable = engine.getDelegatedTask()) != null) {
+                log("    running delegated task...");
+                runnable.run();
+            }
+            HandshakeStatus hsStatus = engine.getHandshakeStatus();
+            if (hsStatus == HandshakeStatus.NEED_TASK) {
+                throw new Exception(
+                    "handshake shouldn't need additional tasks");
+            }
+            logEngineStatus(engine);
+        }
+    }
+
+    private static boolean isEngineClosed(SSLEngine engine) {
+        return (engine.isOutboundDone() && engine.isInboundDone());
+    }
+
+    /*
+     * Simple check to make sure everything came across as expected.
+     */
+    private static void checkTransfer(ByteBuffer a, ByteBuffer b)
+            throws Exception {
+        a.flip();
+        b.flip();
+
+        if (!a.equals(b)) {
+            throw new Exception("Data didn't transfer cleanly");
+        } else {
+            log("\tData transferred cleanly");
+        }
+
+        a.position(a.limit());
+        b.position(b.limit());
+        a.limit(a.capacity());
+        b.limit(b.capacity());
+    }
+
+    /*
+     * Logging code
+     */
+    private static boolean resultOnce = true;
+
+    private static void log(String str, SSLEngineResult result) {
+        if (!logging) {
+            return;
+        }
+        if (resultOnce) {
+            resultOnce = false;
+            System.out.println("The format of the SSLEngineResult is: \n" +
+                    "\t\"getStatus() / getHandshakeStatus()\" +\n" +
+                    "\t\"bytesConsumed() / bytesProduced()\"\n");
+        }
+        HandshakeStatus hsStatus = result.getHandshakeStatus();
+        log(str +
+                result.getStatus() + "/" + hsStatus + ", " +
+                result.bytesConsumed() + "/" + result.bytesProduced() +
+                " bytes");
+        if (hsStatus == HandshakeStatus.FINISHED) {
+            log("\t...ready for application data");
+        }
+    }
+
+    private static void log(String str) {
+        if (logging) {
+            System.out.println(str);
+        }
+    }
+}