8268205: Enhance DTLS client handshake

Reviewed-by: jnimeh, ahgross, rhalade

src/java.base/share/classes/sun/security/ssl/DTLSInputRecord.java

@@ -567,6 +567,9 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
 
         HashMap<Byte, List<HoleDescriptor>> holesMap;
 
+        // A map used to check duplicated handshake messages.
+        HashMap<Byte, Integer> messageSeqMap;
+
         HandshakeFlight() {
             this.handshakeType = HF_UNKNOWN;
             this.flightEpoch = 0;
@@ -577,6 +580,7 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
             this.maxRecordSeq = -1;
 
             this.holesMap = new HashMap<>(5);
+            this.messageSeqMap = new HashMap<>(5);
         }
 
         boolean isRetransmitOf(HandshakeFlight hs) {
@@ -598,6 +602,7 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
             hf.maxRecordSeq = this.maxRecordSeq;
 
             hf.holesMap = new HashMap<>(this.holesMap);
+            hf.messageSeqMap = new HashMap<>(this.messageSeqMap);
 
             return hf;
         }
@@ -640,7 +645,7 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
         }
 
         // Queue up a handshake message.
-        void queueUpHandshake(HandshakeFragment hsf) {
+        void queueUpHandshake(HandshakeFragment hsf) throws SSLProtocolException {
             if (!isDesirable(hsf)) {
                 // Not a dedired record, discard it.
                 return;
@@ -707,6 +712,7 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
                     holes.add(new HoleDescriptor(0, hsf.messageLength));
                 }
                 handshakeFlight.holesMap.put(hsf.handshakeType, holes);
+                handshakeFlight.messageSeqMap.put(hsf.handshakeType, hsf.messageSeq);
             } else if (holes.isEmpty()) {
                 // Have got the full handshake message.  This record may be
                 // a handshake message retransmission.  Discard this record.
@@ -778,7 +784,8 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
         }
 
         // Queue up a ChangeCipherSpec message
-        void queueUpChangeCipherSpec(RecordFragment rf) {
+        void queueUpChangeCipherSpec(RecordFragment rf)
+                throws SSLProtocolException {
             if (!isDesirable(rf)) {
                 // Not a dedired record, discard it.
                 return;
@@ -807,7 +814,7 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
         // Queue up a ciphertext message.
         //
         // Note: not yet be able to decrypt the message.
-        void queueUpFragment(RecordFragment rf) {
+        void queueUpFragment(RecordFragment rf) throws SSLProtocolException {
             if (!isDesirable(rf)) {
                 // Not a dedired record, discard it.
                 return;
@@ -895,7 +902,7 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
         // Is a desired record?
         //
         // Check for retransmission and lost records.
-        private boolean isDesirable(RecordFragment rf) {
+        private boolean isDesirable(RecordFragment rf) throws SSLProtocolException {
             //
             // Discard records old than the previous epoch.
             //
@@ -970,6 +977,25 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
                 return false;
             }
 
+            // Unexpected duplicated handshake messages.
+            if (rf.recordEpoch == handshakeEpoch &&
+                    // For handshake messages only.
+                    rf instanceof HandshakeFragment hsf &&
+                    // Check on the received handshake messages.
+                    handshakeFlight.holesMap.containsKey(hsf.handshakeType)) {
+                Integer cachedMsgSeq = handshakeFlight.messageSeqMap.get(
+                        hsf.handshakeType);
+                if (cachedMsgSeq != null && cachedMsgSeq != hsf.messageSeq) {
+                    // Handshake messages of the same type but with different
+                    // message sequence numbers are not allowed.
+                    throw new SSLProtocolException(
+                            "Two message sequence numbers are used for the "
+                          + "same handshake message ("
+                          + SSLHandshake.nameOf(hsf.handshakeType)
+                          + ")");
+                }
+            }
+
             return true;
         }
 
@@ -1086,6 +1112,9 @@ final class DTLSInputRecord extends InputRecord implements DTLSRecord {
             // cleanup holes map
             handshakeFlight.holesMap.clear();
 
+            // cleanup handshake message sequence numbers map
+            handshakeFlight.messageSeqMap.clear();
+
             // Ready to accept new input record.
             flightIsReady = false;
             needToCheckFlight = false;