8244634: LoadLibraryW failed from tools/jpackage tests after JDK-8242302

Reviewed-by: herrick, almatvee

src/jdk.incubator.jpackage/share/native/applauncher/JvmLauncher.h

@@ -46,6 +46,10 @@ public:
         return *this;
     }
 
+    tstring getPath() const {
+        return jvmPath;
+    }
+
     void launch();
 
 private:

src/jdk.incubator.jpackage/windows/native/applauncher/WinLauncher.cpp

@@ -28,10 +28,14 @@
 #include <windows.h>
 
 #include "AppLauncher.h"
+#include "JvmLauncher.h"
 #include "Log.h"
+#include "Dll.h"
+#include "Toolbox.h"
 #include "FileUtils.h"
 #include "UniqueHandle.h"
 #include "ErrorHandling.h"
+#include "WinSysInfo.h"
 #include "WinErrorHandling.h"
 
 
@@ -41,6 +45,61 @@
 
 namespace {
 
+std::unique_ptr<Dll> loadDllWithAlteredPATH(const tstring& dllFullPath) {
+    LOG_TRACE_FUNCTION();
+
+    const tstring vanillaPathEnvVariable = SysInfo::getEnvVariable(_T("PATH"));
+
+    tstring pathEnvVariable = vanillaPathEnvVariable
+            + _T(";")
+            + FileUtils::dirname(dllFullPath);
+
+    SysInfo::setEnvVariable(_T("PATH"), pathEnvVariable);
+
+    LOG_TRACE(tstrings::any() << "New value of PATH: " << pathEnvVariable);
+
+    // Schedule restore of PATH after attempt to load the given dll
+    const auto resetPATH = runAtEndOfScope([&vanillaPathEnvVariable]() -> void {
+        SysInfo::setEnvVariable(_T("PATH"), vanillaPathEnvVariable);
+    });
+
+    return std::unique_ptr<Dll>(new Dll(dllFullPath));
+}
+
+std::unique_ptr<Dll> loadDllWithAddDllDirectory(const tstring& dllFullPath) {
+    LOG_TRACE_FUNCTION();
+
+    const tstring dirPath = FileUtils::dirname(dllFullPath);
+
+    typedef DLL_DIRECTORY_COOKIE(WINAPI *AddDllDirectoryFunc)(PCWSTR);
+
+    DllFunction<AddDllDirectoryFunc> _AddDllDirectory(
+            Dll("kernel32.dll", Dll::System()), "AddDllDirectory");
+
+    AddDllDirectoryFunc func = _AddDllDirectory;
+    DLL_DIRECTORY_COOKIE res = func(dirPath.c_str());
+    if (!res) {
+        JP_THROW(SysError(tstrings::any()
+                << "AddDllDirectory(" << dirPath << ") failed", func));
+    }
+
+    LOG_TRACE(tstrings::any() << "AddDllDirectory(" << dirPath << "): OK");
+
+    // Important: use LOAD_LIBRARY_SEARCH_DEFAULT_DIRS flag,
+    // but not LOAD_LIBRARY_SEARCH_USER_DIRS!
+    HMODULE dllHandle = LoadLibraryEx(dllFullPath.c_str(), NULL,
+            LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+
+    LOG_TRACE(tstrings::any() << "LoadLibraryEx(" << dllFullPath
+            << ", LOAD_LIBRARY_SEARCH_DEFAULT_DIRS): " << dllHandle);
+
+    const auto freeDll = runAtEndOfScope([&dllHandle]() -> void {
+        Dll::freeLibrary(dllHandle);
+    });
+
+    return std::unique_ptr<Dll>(new Dll(dllFullPath));
+}
+
 void launchApp() {
     // [RT-31061] otherwise UI can be left in back of other windows.
     ::AllowSetForegroundWindow(ASFW_ANY);
@@ -48,13 +107,31 @@ void launchApp() {
     const tstring launcherPath = SysInfo::getProcessModulePath();
     const tstring appImageRoot = FileUtils::dirname(launcherPath);
 
-    AppLauncher()
+    std::unique_ptr<Jvm> jvm(AppLauncher()
         .setImageRoot(appImageRoot)
         .addJvmLibName(_T("bin\\jli.dll"))
         .setAppDir(FileUtils::mkpath() << appImageRoot << _T("app"))
         .setDefaultRuntimePath(FileUtils::mkpath() << appImageRoot
                 << _T("runtime"))
-        .launch();
+        .createJvmLauncher());
+
+    std::unique_ptr<Dll> jvmDll;
+    try {
+        // Try load JVM DLL.
+        jvmDll = std::unique_ptr<Dll>(new Dll(jvm->getPath()));
+    } catch (const std::exception&) {
+        // JVM DLL load failed, though it exists in file system.
+        try {
+            // Try adjust the DLL search paths with AddDllDirectory() WINAPI CALL
+            jvmDll = loadDllWithAddDllDirectory(jvm->getPath());
+        } catch (const std::exception&) {
+            // AddDllDirectory() didn't work. Try altering PATH environment
+            // variable as the last resort.
+            jvmDll = loadDllWithAlteredPATH(jvm->getPath());
+        }
+    }
+
+    jvm->launch();
 }
 
 } // namespace

src/jdk.incubator.jpackage/windows/native/common/WinSysInfo.cpp

@@ -114,6 +114,16 @@ HMODULE getCurrentModuleHandle()
     return hmodule;
 }
 
+void setEnvVariable(const tstring& name, const tstring& value)
+{
+    if (!SetEnvironmentVariable(name.c_str(), value.c_str())) {
+        JP_THROW(SysError(tstrings::any()
+                << "SetEnvironmentVariable("
+                << name << ", " << value
+                << ") failed", SetEnvironmentVariable));
+    }
+}
+
 tstring getCurrentModulePath()
 {
     return getModulePath(getCurrentModuleHandle());

src/jdk.incubator.jpackage/windows/native/common/WinSysInfo.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -43,6 +43,8 @@ namespace SysInfo {
     // Returns handle of the current module (exe or dll).
     // The function assumes this code is statically linked to the module.
     HMODULE getCurrentModuleHandle();
+
+    void setEnvVariable(const tstring& name, const tstring& value);
 }