8156020: 8145547 breaks AIX and and uses RTLD_NOLOAD incorrectly

Reviewed-by: prr, serb

jdk/src/java.desktop/unix/native/libawt_xawt/awt/gtk2_interface.c

@@ -312,7 +312,7 @@ static void* dl_symbol_gthread(const char* name)
     return result;
 }
 
-gboolean gtk2_check(const char* lib_name, int flags)
+gboolean gtk2_check(const char* lib_name, gboolean load)
 {
     if (gtk2_libhandle != NULL) {
         /* We've already successfully opened the GTK libs, so return true. */
@@ -320,16 +320,25 @@ gboolean gtk2_check(const char* lib_name, int flags)
     } else {
         void *lib = NULL;
 
-        lib = dlopen(lib_name, flags);
+#ifdef RTLD_NOLOAD
+        /* Just check if gtk libs are already in the process space */
+        lib = dlopen(lib_name, RTLD_LAZY | RTLD_NOLOAD);
+        if (!load || lib != NULL) {
+            return lib != NULL;
+        }
+#else
+#ifdef _AIX
+        /* On AIX we could implement this with the help of loadquery(L_GETINFO, ..)  */
+        /* (see reload_table() in hotspot/src/os/aix/vm/loadlib_aix.cpp) but it is   */
+        /* probably not worth it because most AIX servers don't have GTK libs anyway */
+#endif
+#endif
 
+        lib = dlopen(lib_name, RTLD_LAZY | RTLD_LOCAL);
         if (lib == NULL) {
             return FALSE;
         }
 
-        if (flags & RTLD_NOLOAD) {
-            return TRUE;
-        }
-
         fp_gtk_check_version = dlsym(lib, "gtk_check_version");
         /* Check for GTK 2.2+ */
         if (!fp_gtk_check_version(2, 2, 0)) {

jdk/src/java.desktop/unix/native/libawt_xawt/awt/gtk3_interface.c

@@ -87,13 +87,25 @@ static void* dl_symbol(const char* name)
     return result;
 }
 
-gboolean gtk3_check(const char* lib_name, int flags)
+gboolean gtk3_check(const char* lib_name, gboolean load)
 {
     if (gtk3_libhandle != NULL) {
         /* We've already successfully opened the GTK libs, so return true. */
         return TRUE;
     } else {
-        return dlopen(lib_name, flags) != NULL;
+#ifdef RTLD_NOLOAD
+        void *lib = dlopen(lib_name, RTLD_LAZY | RTLD_NOLOAD);
+        if (!load || lib != NULL) {
+            return lib != NULL;
+        }
+#else
+#ifdef _AIX
+        /* On AIX we could implement this with the help of loadquery(L_GETINFO, ..)  */
+        /* (see reload_table() in hotspot/src/os/aix/vm/loadlib_aix.cpp) but it is   */
+        /* probably not worth it because most AIX servers don't have GTK libs anyway */
+#endif
+#endif
+        return dlopen(lib_name, RTLD_LAZY | RTLD_LOCAL) != NULL;
     }
 }
 

jdk/src/java.desktop/unix/native/libawt_xawt/awt/gtk_interface.c

@@ -30,8 +30,8 @@
 GtkApi* gtk2_load(JNIEnv *env, const char* lib_name);
 GtkApi* gtk3_load(JNIEnv *env, const char* lib_name);
 
-gboolean gtk2_check(const char* lib_name, int flags);
-gboolean gtk3_check(const char* lib_name, int flags);
+gboolean gtk2_check(const char* lib_name, gboolean load);
+gboolean gtk3_check(const char* lib_name, gboolean load);
 
 GtkApi *gtk;
 
@@ -40,7 +40,7 @@ typedef struct {
     const char* name;
     const char* vname;
     GtkApi* (*load)(JNIEnv *env, const char* lib_name);
-    gboolean (*check)(const char* lib_name, int flags);
+    gboolean (*check)(const char* lib_name, gboolean load);
 } GtkLib;
 
 static GtkLib libs[] = {
@@ -70,10 +70,10 @@ static GtkLib libs[] = {
 static GtkLib* get_loaded() {
     GtkLib* lib = libs;
     while(!gtk && lib->version) {
-        if (lib->check(lib->vname, RTLD_NOLOAD)) {
+        if (lib->check(lib->vname, /* load = */FALSE)) {
             return lib;
         }
-        if (lib->check(lib->name, RTLD_NOLOAD)) {
+        if (lib->check(lib->name, /* load = */FALSE)) {
             return lib;
         }
         lib++;
@@ -130,14 +130,14 @@ gboolean gtk_load(JNIEnv *env, GtkVersion version, gboolean verbose) {
     return gtk != NULL;
 }
 
-static gboolean check_version(GtkVersion version, int flags) {
+static gboolean check_version(GtkVersion version) {
     GtkLib* lib = libs;
     while (lib->version) {
         if (version == GTK_ANY || lib->version == version) {
-            if (lib->check(lib->vname, flags)) {
+            if (lib->check(lib->vname, /* load = */TRUE)) {
                 return TRUE;
             }
-            if (lib->check(lib->name, flags)) {
+            if (lib->check(lib->name, /* load = */TRUE)) {
                 return TRUE;
             }
         }
@@ -150,9 +150,6 @@ gboolean gtk_check_version(GtkVersion version) {
     if (gtk) {
         return TRUE;
     }
-    if (check_version(version, RTLD_NOLOAD)) {
-        return TRUE;
-    }
-    return check_version(version, RTLD_LAZY | RTLD_LOCAL);
+    return check_version(version);
 }