8195607: sun/security/pkcs11/Secmod/TestNssDbSqlite.java failed with "NSS initialization failed" on NSS 3.34.1

Reviewed-by: valeriep, weijun

src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/Secmod.java

@@ -197,7 +197,7 @@ public final class Secmod {
 
         if (configDir != null) {
             String configDirPath = null;
-            String sqlPrefix = "sql:/";
+            String sqlPrefix = "sql:";
             if (!configDir.startsWith(sqlPrefix)) {
                 configDirPath = configDir;
             } else {

src/jdk.crypto.cryptoki/share/native/libj2pkcs11/j2secmod.c

@@ -69,9 +69,14 @@ JNIEXPORT jboolean JNICALL Java_sun_security_pkcs11_Secmod_nssInitialize
     int res = 0;
     FPTR_Initialize initialize =
         (FPTR_Initialize)findFunction(env, jHandle, "NSS_Initialize");
+    #ifdef SECMOD_DEBUG
+    FPTR_GetError getError =
+        (FPTR_GetError)findFunction(env, jHandle, "PORT_GetError");
+    #endif // SECMOD_DEBUG
     unsigned int flags = 0x00;
     const char *configDir = NULL;
     const char *functionName = NULL;
+    const char *configFile = NULL;
 
     /* If we cannot initialize, exit now */
     if (initialize == NULL) {
@@ -97,13 +102,18 @@ JNIEXPORT jboolean JNICALL Java_sun_security_pkcs11_Secmod_nssInitialize
         flags = 0x20; // NSS_INIT_OPTIMIZESPACE flag
     }
 
+    configFile = "secmod.db";
+    if (configDir != NULL && strncmp("sql:", configDir, 4U) == 0) {
+        configFile = "pkcs11.txt";
+    }
+
     /*
      * If the NSS_Init function is requested then call NSS_Initialize to
      * open the Cert, Key and Security Module databases, read only.
      */
     if (strcmp("NSS_Init", functionName) == 0) {
         flags = flags | 0x01; // NSS_INIT_READONLY flag
-        res = initialize(configDir, "", "", "secmod.db", flags);
+        res = initialize(configDir, "", "", configFile, flags);
 
     /*
      * If the NSS_InitReadWrite function is requested then call
@@ -111,7 +121,7 @@ JNIEXPORT jboolean JNICALL Java_sun_security_pkcs11_Secmod_nssInitialize
      * read/write.
      */
     } else if (strcmp("NSS_InitReadWrite", functionName) == 0) {
-        res = initialize(configDir, "", "", "secmod.db", flags);
+        res = initialize(configDir, "", "", configFile, flags);
 
     /*
      * If the NSS_NoDB_Init function is requested then call
@@ -137,6 +147,13 @@ cleanup:
         (*env)->ReleaseStringUTFChars(env, jConfigDir, configDir);
     }
     dprintf1("-res: %d\n", res);
+    #ifdef SECMOD_DEBUG
+    if (res == -1) {
+        if (getError != NULL) {
+            dprintf1("-NSS error: %d\n", getError());
+        }
+    }
+    #endif // SECMOD_DEBUG
 
     return (res == 0) ? JNI_TRUE : JNI_FALSE;
 }

src/jdk.crypto.cryptoki/unix/native/libj2pkcs11/j2secmod_md.h

@@ -34,6 +34,10 @@ typedef int (*FPTR_Initialize)(const char *configdir,
         const char *certPrefix, const char *keyPrefix,
         const char *secmodName, unsigned int flags);
 
+#ifdef SECMOD_DEBUG
+typedef int (*FPTR_GetError)(void);
+#endif //SECMOD_DEBUG
+
 // in secmod.h
 //extern SECMODModule *SECMOD_LoadModule(char *moduleSpec,SECMODModule *parent,
 //                                                      PRBool recurse);

test/jdk/sun/security/pkcs11/Secmod/pkcs11.txt

@@ -0,0 +1,4 @@
+library=
+name=NSS Internal PKCS #11 Module
+parameters=configdir='sql:./tmpdb' certPrefix='' keyPrefix='' secmod='' flags= updatedir='' updateCertPrefix='' updateKeyPrefix='' updateid='' updateTokenDescription='' 
+NSS=Flags=internal,critical trustOrder=75 cipherOrder=100 slotParams=(1={slotFlags=[RSA,DSA,DH,RC2,RC4,DES,RANDOM,SHA1,MD5,MD2,SSL,TLS,AES,Camellia,SEED,SHA256,SHA512] askpw=any timeout=30})

test/jdk/sun/security/pkcs11/SecmodTest.java

@@ -55,7 +55,7 @@ public class SecmodTest extends PKCS11Test {
 
         DBDIR = System.getProperty("test.classes", ".") + SEP + "tmpdb";
         if (useSqlite) {
-            System.setProperty("pkcs11test.nss.db", "sql:/" + DBDIR);
+            System.setProperty("pkcs11test.nss.db", "sql:" + DBDIR);
         } else {
             System.setProperty("pkcs11test.nss.db", DBDIR);
         }
@@ -67,6 +67,7 @@ public class SecmodTest extends PKCS11Test {
         if (useSqlite) {
             copyFile("key4.db", BASE, DBDIR);
             copyFile("cert9.db", BASE, DBDIR);
+            copyFile("pkcs11.txt", BASE, DBDIR);
         } else {
             copyFile("secmod.db", BASE, DBDIR);
             copyFile("key3.db", BASE, DBDIR);