8074580: sun/security/pkcs11/rsa/TestKeyPairGenerator.java fails due to PKCS11Exception: CKR_FUNCTION_FAILED

Reviewed-by: valeriep
This commit is contained in:
Artem Smotrakov 2016-06-22 15:58:08 -07:00
parent e922fb0afb
commit f2b622bfb3
7 changed files with 65 additions and 16 deletions

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
*/
/* Copyright (c) 2002 Graz University of Technology. All rights reserved.
@ -54,6 +54,8 @@ import java.util.*;
import java.security.AccessController;
import java.security.PrivilegedAction;
import sun.security.util.Debug;
import static sun.security.pkcs11.wrapper.PKCS11Constants.*;
/**
@ -85,7 +87,8 @@ public class PKCS11 {
return null;
}
});
initializeLibrary();
boolean enableDebug = Debug.getInstance("sunpkcs11") != null;
initializeLibrary(enableDebug);
}
public static void loadNative() {
@ -109,7 +112,7 @@ public class PKCS11 {
* @preconditions
* @postconditions
*/
private static native void initializeLibrary();
private static native void initializeLibrary(boolean debug);
// XXX
/**

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
*/
/* Copyright (c) 2002 Graz University of Technology. All rights reserved.
@ -73,6 +73,8 @@ jclass jLongClass;
JavaVM* jvm = NULL;
jboolean debug = 0;
JNIEXPORT jint JNICALL DEF_JNI_OnLoad(JavaVM *vm, void *reserved) {
jvm = vm;
return JNI_VERSION_1_4;
@ -92,7 +94,7 @@ JNIEXPORT jint JNICALL DEF_JNI_OnLoad(JavaVM *vm, void *reserved) {
*/
JNIEXPORT void JNICALL
Java_sun_security_pkcs11_wrapper_PKCS11_initializeLibrary
(JNIEnv *env, jclass thisClass)
(JNIEnv *env, jclass thisClass, jboolean enableDebug)
{
#ifndef NO_CALLBACKS
if (notifyListLock == NULL) {
@ -101,6 +103,7 @@ Java_sun_security_pkcs11_wrapper_PKCS11_initializeLibrary
#endif
prefetchFields(env, thisClass);
debug = enableDebug;
}
jclass fetchClass(JNIEnv *env, const char *name) {

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
*/
/* Copyright (c) 2002 Graz University of Technology. All rights reserved.
@ -152,6 +152,8 @@ JNIEXPORT jlongArray JNICALL Java_sun_security_pkcs11_wrapper_PKCS11_C_1Generate
CK_OBJECT_HANDLE_PTR ckpKeyHandles; /* pointer to array with Public and Private Key */
jlongArray jKeyHandles = NULL;
CK_RV rv;
int attempts;
const int MAX_ATTEMPTS = 3;
CK_FUNCTION_LIST_PTR ckpFunctions = getFunctionList(env, obj);
if (ckpFunctions == NULL) { return NULL; }
@ -190,10 +192,35 @@ JNIEXPORT jlongArray JNICALL Java_sun_security_pkcs11_wrapper_PKCS11_C_1Generate
return NULL;
}
rv = (*ckpFunctions->C_GenerateKeyPair)(ckSessionHandle, &ckMechanism,
ckpPublicKeyAttributes, ckPublicKeyAttributesLength,
ckpPrivateKeyAttributes, ckPrivateKeyAttributesLength,
ckpPublicKeyHandle, ckpPrivateKeyHandle);
/*
* Workaround for NSS bug 1012786:
*
* Key generation may fail with CKR_FUNCTION_FAILED error
* if there is insufficient entropy to generate a random key.
*
* PKCS11 spec says the following about CKR_FUNCTION_FAILED error
* (see section 11.1.1):
*
* ... In any event, although the function call failed, the situation
* is not necessarily totally hopeless, as it is likely to be
* when CKR_GENERAL_ERROR is returned. Depending on what the root cause of
* the error actually was, it is possible that an attempt
* to make the exact same function call again would succeed.
*
* Call C_GenerateKeyPair() several times if CKR_FUNCTION_FAILED occurs.
*/
for (attempts = 0; attempts < MAX_ATTEMPTS; attempts++) {
rv = (*ckpFunctions->C_GenerateKeyPair)(ckSessionHandle, &ckMechanism,
ckpPublicKeyAttributes, ckPublicKeyAttributesLength,
ckpPrivateKeyAttributes, ckPrivateKeyAttributesLength,
ckpPublicKeyHandle, ckpPrivateKeyHandle);
if (rv == CKR_FUNCTION_FAILED) {
printDebug("C_1GenerateKeyPair(): C_GenerateKeyPair() failed \
with CKR_FUNCTION_FAILED error, try again\n");
} else {
break;
}
}
if (ckAssertReturnValueOK(env, rv) == CK_ASSERT_OK) {
jKeyHandles = ckULongArrayToJLongArray(env, ckpKeyHandles, 2);

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
*/
/* Copyright (c) 2002 Graz University of Technology. All rights reserved.
@ -1143,3 +1143,15 @@ void p11free(void *p, char *file, int line) {
#endif
// prints a message to stdout if debug output is enabled
void printDebug(const char *format, ...) {
if (debug == JNI_TRUE) {
va_list args;
fprintf(stdout, "sunpkcs11: ");
va_start(args, format);
vfprintf(stdout, format, args);
va_end(args);
fflush(stdout);
}
}

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
*/
/* Copyright (c) 2002 Graz University of Technology. All rights reserved.
@ -159,6 +159,7 @@
#include "pkcs-11v2-20a3.h"
#include <jni.h>
#include <jni_util.h>
#include <stdarg.h>
#define MAX_STACK_BUFFER_LEN (4 * 1024)
#define MAX_HEAP_BUFFER_LEN (64 * 1024)
@ -217,6 +218,10 @@
#define TRACE_UNINTEND
#endif
/* debug output */
extern jboolean debug;
void printDebug(const char *format, ...);
#define CK_ASSERT_OK 0L
#define CLASS_INFO "sun/security/pkcs11/wrapper/CK_INFO"

View File

@ -272,8 +272,6 @@ sun/security/pkcs11/tls/TestMasterSecret.java 8077138,8023434
sun/security/pkcs11/tls/TestPRF.java 8077138,8023434 windows-all
sun/security/pkcs11/tls/TestPremaster.java 8077138,8023434 windows-all
sun/security/pkcs11/rsa/TestKeyPairGenerator.java 8074580 generic-all
sun/security/krb5/auto/HttpNegotiateServer.java 8038079 generic-all
sun/security/tools/keytool/autotest.sh 8130302 generic-all

View File

@ -29,8 +29,9 @@
* @library ..
* @library /lib/testlibrary
* @build jdk.testlibrary.*
* @run main/othervm TestKeyPairGenerator
* @run main/othervm TestKeyPairGenerator sm TestKeyPairGenerator.policy
* @run main/othervm -Djava.security.debug=sunpkcs11 TestKeyPairGenerator
* @run main/othervm -Djava.security.debug=sunpkcs11 TestKeyPairGenerator
* sm TestKeyPairGenerator.policy
* @key intermittent randomness
*/