8271566: DSA signature length value is not accurate in P11Signature

Reviewed-by: valeriep

src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11Signature.java

@@ -119,6 +119,9 @@ final class P11Signature extends SignatureSpi {
     // key instance used, if init*() was called
     private P11Key p11Key;
 
+    // signature length expected or 0 for unknown
+    private int sigLen;
+
     // message digest, if we do the digesting ourselves
     private final MessageDigest md;
 
@@ -285,7 +288,7 @@ final class P11Signature extends SignatureSpi {
         try {
             if (mode == M_SIGN) {
                 if (type == T_UPDATE) {
-                    token.p11.C_SignFinal(session.id(), 0);
+                    token.p11.C_SignFinal(session.id(), sigLen);
                 } else {
                     byte[] digest;
                     if (type == T_DIGEST) {
@@ -296,12 +299,7 @@ final class P11Signature extends SignatureSpi {
                     token.p11.C_Sign(session.id(), digest);
                 }
             } else { // M_VERIFY
-                byte[] signature;
+                byte[] signature = new byte[sigLen];
-                if (mechanism == CKM_DSA) {
-                    signature = new byte[64]; // assume N = 256
-                } else {
-                    signature = new byte[(p11Key.length() + 7) >> 3];
-                }
                 if (type == T_UPDATE) {
                     token.p11.C_VerifyFinal(session.id(), signature);
                 } else {
@@ -372,6 +370,15 @@ final class P11Signature extends SignatureSpi {
                 md.reset();
             }
         }
+        sigLen = 0;
+        if ("DSA".equals(p11Key.getAlgorithm())) {
+            if (p11Key instanceof P11Key.P11DSAPrivateKeyInternal) {
+                sigLen = ((P11Key.P11DSAPrivateKeyInternal)p11Key).getParams()
+                        .getQ().bitLength() >> 2;
+            } else if (p11Key instanceof DSAKey) {
+                sigLen = ((DSAKey)p11Key).getParams().getQ().bitLength() >> 2;
+            }
+        }
         initialized = true;
     }
 
@@ -617,7 +624,7 @@ final class P11Signature extends SignatureSpi {
         try {
             byte[] signature;
             if (type == T_UPDATE) {
-                signature = token.p11.C_SignFinal(session.id(), 0);
+                signature = token.p11.C_SignFinal(session.id(), sigLen);
             } else {
                 byte[] digest;
                 if (type == T_DIGEST) {
@@ -684,7 +691,7 @@ final class P11Signature extends SignatureSpi {
         try {
             if (!p1363Format) {
                 if (keyAlgorithm.equals("DSA")) {
-                    signature = asn1ToDSA(signature);
+                    signature = asn1ToDSA(signature, sigLen);
                 } else if (keyAlgorithm.equals("EC")) {
                     signature = asn1ToECDSA(signature);
                 }
@@ -801,7 +808,8 @@ final class P11Signature extends SignatureSpi {
         }
     }
 
-    private static byte[] asn1ToDSA(byte[] sig) throws SignatureException {
+    private static byte[] asn1ToDSA(byte[] sig, int sigLen)
+            throws SignatureException {
         try {
             // Enforce strict DER checking for signatures
             DerInputStream in = new DerInputStream(sig, 0, sig.length, false);
@@ -816,8 +824,8 @@ final class P11Signature extends SignatureSpi {
             BigInteger r = values[0].getPositiveBigInteger();
             BigInteger s = values[1].getPositiveBigInteger();
 
-            byte[] br = toByteArray(r, 20);
+            byte[] br = toByteArray(r, sigLen/2);
-            byte[] bs = toByteArray(s, 20);
+            byte[] bs = toByteArray(s, sigLen/2);
             if ((br == null) || (bs == null)) {
                 throw new SignatureException("Out of range value for R or S");
             }
@@ -829,7 +837,7 @@ final class P11Signature extends SignatureSpi {
         }
     }
 
-    private byte[] asn1ToECDSA(byte[] sig) throws SignatureException {
+    private static byte[] asn1ToECDSA(byte[] sig) throws SignatureException {
         try {
             // Enforce strict DER checking for signatures
             DerInputStream in = new DerInputStream(sig, 0, sig.length, false);
@@ -909,3 +917,4 @@ final class P11Signature extends SignatureSpi {
         return null;
     }
 }
+

test/jdk/sun/security/pkcs11/Signature/LargeDSAKey.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2021, Red Hat, Inc.
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.security.AlgorithmParameterGenerator;
+import java.security.AlgorithmParameters;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.PrivateKey;
+import java.security.Provider;
+import java.security.PublicKey;
+import java.security.SecureRandom;
+import java.security.Signature;
+import java.security.spec.DSAGenParameterSpec;
+import java.security.spec.DSAParameterSpec;
+
+/*
+ * @test
+ * @bug 8271566
+ * @library /test/lib ..
+ * @modules jdk.crypto.cryptoki
+ * @run main/othervm/timeout=30 LargeDSAKey
+ */
+
+public final class LargeDSAKey extends PKCS11Test {
+
+    private static final boolean enableDebug = false;
+
+    private static final String knownText =
+            "Known text known text known text";
+
+    @Override
+    public void main(Provider p) throws Exception {
+        KeyPairGenerator kpg = KeyPairGenerator.getInstance("DSA", p);
+        AlgorithmParameterGenerator dsaParGen =
+                AlgorithmParameterGenerator.getInstance("DSA");
+        DSAGenParameterSpec dsaParGenSpec =
+                new DSAGenParameterSpec(2048, 256);
+        dsaParGen.init(dsaParGenSpec, new SecureRandom());
+        AlgorithmParameters params = dsaParGen.generateParameters();
+        DSAParameterSpec dsaParams =
+                params.getParameterSpec(DSAParameterSpec.class);
+        kpg.initialize(dsaParams);
+        KeyPair kp = kpg.generateKeyPair();
+        doTestSignature(kp, p);
+    }
+
+    private static void doTestSignature(KeyPair kp, Provider p)
+            throws Exception {
+        byte[] knownTextSig = null;
+        Signature s = Signature.getInstance("SHA1withDSA", p);
+        PrivateKey privKey = kp.getPrivate();
+        PublicKey pubKey = kp.getPublic();
+        if (enableDebug) {
+            System.out.println("Signature algorithm: " + s.getAlgorithm());
+            System.out.println("Signature Provider: " + s.getProvider());
+            System.out.println("Private key for signature: " + privKey);
+            System.out.println("Public key for signature: " + pubKey);
+        }
+        s.initSign(privKey);
+        s.update(knownText.getBytes());
+        knownTextSig = s.sign();
+        s.initVerify(pubKey);
+        s.update(knownText.getBytes());
+        if (s.verify(knownTextSig) == false) {
+            throw new Exception("Could not verify signature");
+        }
+        if (enableDebug) {
+            System.out.println("Signature verified");
+        }
+    }
+
+    public static void main(String[] args) throws Throwable {
+        main(new LargeDSAKey());
+        System.out.println("TEST PASS - OK");
+    }
+
+}