8299329: Assertion failure with fastdebug build when trying to use CDS without classpath

Reviewed-by: iklam, ccheung

src/hotspot/share/cds/filemap.cpp

@@ -880,15 +880,17 @@ unsigned int FileMapInfo::longest_common_app_classpath_prefix_len(int num_paths,
       if (rp_array->at(i)[pos] != '\0' && rp_array->at(i)[pos] == rp_array->at(0)[pos]) {
         continue;
       }
-
       // search backward for the pos before the file separator char
-      while (pos > 0 && rp_array->at(0)[--pos] != *os::file_separator());
+      while (pos > 0) {
-      // return the file separator char position
+        if (rp_array->at(0)[--pos] == *os::file_separator()) {
           return pos + 1;
         }
       }
       return 0;
     }
+  }
+  return 0;
+}
 
 bool FileMapInfo::check_paths(int shared_path_start_idx, int num_paths, GrowableArray<const char*>* rp_array,
                               unsigned int dumptime_prefix_len, unsigned int runtime_prefix_len) {
@@ -1022,8 +1024,12 @@ bool FileMapInfo::validate_app_class_paths(int shared_app_paths_len) {
       //     java -Xshare:auto -cp /x/y/Foo.jar:/x/y/b/Bar.jar  ...
       unsigned int dumptime_prefix_len = header()->common_app_classpath_prefix_size();
       unsigned int runtime_prefix_len = longest_common_app_classpath_prefix_len(shared_app_paths_len, rp_array);
+      if (dumptime_prefix_len != 0 || runtime_prefix_len != 0) {
+        log_info(class, path)("LCP length for app classpath (dumptime: %u, runtime: %u)",
+                              dumptime_prefix_len, runtime_prefix_len);
         mismatch = check_paths(j, shared_app_paths_len, rp_array,
                                dumptime_prefix_len, runtime_prefix_len);
+      }
       if (mismatch) {
         return classpath_failure("[APP classpath mismatch, actual: -Djava.class.path=", appcp);
       }

test/hotspot/jtreg/runtime/cds/appcds/WrongClasspath.java

@@ -33,6 +33,9 @@
  */
 
 import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import jdk.test.lib.process.OutputAnalyzer;
 import jdk.test.lib.helpers.ClassFileInstaller;
 
@@ -44,10 +47,26 @@ public class WrongClasspath {
     String mismatchMsg = "shared class paths mismatch";
     String hintMsg = "(hint: enable -Xlog:class+path=info to diagnose the failure)";
 
+    // Dump CDS archive with hello.jar
+    // Run with a jar file that differs from the original jar file by the first character only: -cp mello.jar
+    // Shared class paths mismatch should be detected.
+    String hellojar = "hello.jar";
+    String mellojar = "mello.jar";
+    Files.copy(Paths.get(appJar), Paths.get(hellojar), StandardCopyOption.COPY_ATTRIBUTES);
+    Files.copy(Paths.get(appJar), Paths.get(mellojar), StandardCopyOption.COPY_ATTRIBUTES);
+    TestCommon.testDump(hellojar, TestCommon.list("Hello"));
+    TestCommon.run("-cp", mellojar,
+        "-Xlog:cds",
+        "Hello")
+        .assertAbnormalExit(unableToUseMsg, mismatchMsg, hintMsg);
+
     // Dump an archive with a specified JAR file in -classpath
     TestCommon.testDump(appJar, TestCommon.list("Hello"));
 
     // Then try to execute the archive without -classpath -- it should fail
+    // To run without classpath, set the property test.noclasspath to true
+    // so that ProcessTools won't append the classpath of the jtreg process to the test process
+    System.setProperty("test.noclasspath", "true");
     TestCommon.run(
         /* "-cp", appJar, */ // <- uncomment this and the execution should succeed
         "-Xlog:cds",
@@ -70,6 +89,7 @@ public class WrongClasspath {
                .shouldContain(mismatchMsg)
                .shouldNotContain(hintMsg);
         });
+    System.clearProperty("test.noclasspath");
 
     // Dump CDS archive with 2 jars: -cp hello.jar:jar2.jar
     // Run with 2 jars but the second jar doesn't exist: -cp hello.jarjar2.jarx

test/lib/jdk/test/lib/process/ProcessTools.java

@@ -358,8 +358,12 @@ public final class ProcessTools {
         ArrayList<String> args = new ArrayList<>();
         args.add(javapath);
 
+        String noCPString = System.getProperty("test.noclasspath", "false");
+        boolean noCP = Boolean.valueOf(noCPString);
+        if (!noCP) {
             args.add("-cp");
             args.add(System.getProperty("java.class.path"));
+        }
 
         String mainWrapper = System.getProperty("main.wrapper");
         if (mainWrapper != null) {
@@ -374,7 +378,12 @@ public final class ProcessTools {
             cmdLine.append(cmd).append(' ');
         System.out.println("Command line: [" + cmdLine.toString() + "]");
 
-        return new ProcessBuilder(args);
+        ProcessBuilder pb = new ProcessBuilder(args);
+        if (noCP) {
+            // clear CLASSPATH from the env
+            pb.environment().remove("CLASSPATH");
+        }
+        return pb;
     }
 
     private static void printStack(Thread t, StackTraceElement[] stack) {