8226719: Kerberos login to Windows 2000 failed with "Inappropriate type of checksum in message"

Reviewed-by: xuelei

src/java.security.jgss/share/classes/sun/security/krb5/Checksum.java

@@ -197,6 +197,26 @@ public class Checksum {
             usage);
     }
 
+    // ===============  ATTENTION! Use with care  ==================
+    // According to https://tools.ietf.org/html/rfc3961#section-6.1,
+    // An unkeyed checksum should only be used "in limited circumstances
+    // where the lack of a key does not provide a window for an attack,
+    // preferably as part of an encrypted message".
+    public boolean verifyAnyChecksum(byte[] data, EncryptionKey key,
+            int usage)
+            throws KdcErrException, KrbCryptoException {
+        CksumType cksumEngine = CksumType.getInstance(cksumType);
+        if (!cksumEngine.isSafe()) {
+            return cksumEngine.verifyChecksum(data, checksum);
+        } else {
+            return cksumEngine.verifyKeyedChecksum(data,
+                    data.length,
+                    key.getBytes(),
+                    checksum,
+                    usage);
+        }
+    }
+
     /*
     public Checksum(byte[] data) throws KdcErrException, KrbCryptoException {
         this(Checksum.CKSUMTYPE_DEFAULT, data);

src/java.security.jgss/share/classes/sun/security/krb5/KrbKdcRep.java

@@ -158,8 +158,10 @@ abstract class KrbKdcRep {
                             Checksum repCksum = new Checksum(
                                     new DerInputStream(
                                             pa.getValue()).getDerValue());
+                            // The checksum is inside encKDCRepPart so we don't
+                            // care if it's keyed or not.
                             repPaReqEncPaRepValid =
-                                    repCksum.verifyKeyedChecksum(
+                                    repCksum.verifyAnyChecksum(
                                             req.asn1Encode(), replyKey,
                                             KeyUsage.KU_AS_REQ);
                         } catch (Exception e) {

src/java.security.jgss/share/classes/sun/security/krb5/internal/crypto/CksumType.java

@@ -156,6 +156,11 @@ public abstract class CksumType {
     public abstract byte[] calculateKeyedChecksum(byte[] data, int size,
         byte[] key, int usage) throws KrbCryptoException;
 
+    public boolean verifyChecksum(byte[] data, byte[] checksum)
+            throws KrbCryptoException {
+        throw new UnsupportedOperationException("Not supported");
+    }
+
     public abstract boolean verifyKeyedChecksum(byte[] data, int size,
         byte[] key, byte[] checksum, int usage) throws KrbCryptoException;
 

src/java.security.jgss/share/classes/sun/security/krb5/internal/crypto/RsaMd5CksumType.java

@@ -101,4 +101,14 @@ public final class RsaMd5CksumType extends CksumType {
         return false;
     }
 
+    @Override
+    public boolean verifyChecksum(byte[] data, byte[] checksum)
+            throws KrbCryptoException {
+        try {
+            byte[] calculated = MessageDigest.getInstance("MD5").digest(data);
+            return CksumType.isChecksumEqual(calculated, checksum);
+        } catch (Exception e) {
+            return false;
+        }
+    }
 }