8266345: (fs) Custom DefaultFileSystemProvider security related loops

Co-authored-by: Brian Burkhalter <bpb@openjdk.org>
Reviewed-by: bpb, chegar

src/java.base/share/classes/sun/security/provider/PolicyFile.java

@@ -44,6 +44,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import jdk.internal.access.JavaSecurityAccess;
 import jdk.internal.access.SharedSecrets;
 import jdk.internal.util.StaticProperty;
+import sun.nio.fs.DefaultFileSystemProvider;
 import sun.security.util.*;
 import sun.net.www.ParseUtil;
 
@@ -276,6 +277,13 @@ public class PolicyFile extends java.security.Policy {
     private static Set<URL> badPolicyURLs =
         Collections.newSetFromMap(new ConcurrentHashMap<URL,Boolean>());
 
+    /**
+     * Use the platform's default file system to avoid recursive initialization
+     * issues when the VM is configured to use a custom file system provider.
+     */
+    private static final java.nio.file.FileSystem builtInFS =
+        DefaultFileSystemProvider.theFileSystem();
+
     /**
      * Initializes the Policy object and reads the default policy
      * configuration file(s) into the Policy object.
@@ -475,7 +483,7 @@ public class PolicyFile extends java.security.Policy {
     }
 
     private void initDefaultPolicy(PolicyInfo newInfo) {
-        Path defaultPolicy = Path.of(StaticProperty.javaHome(),
+        Path defaultPolicy = builtInFS.getPath(StaticProperty.javaHome(),
                                      "lib",
                                      "security",
                                      "default.policy");

test/jdk/java/nio/file/spi/SetDefaultProvider.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,7 @@
 
 /**
  * @test
+ * @bug 8266345
  * @modules jdk.jartool
  * @library /test/lib
  * @build SetDefaultProvider TestProvider m/* jdk.test.lib.process.ProcessTools
@@ -72,6 +73,22 @@ public class SetDefaultProvider {
         assertTrue(exitValue == 0);
     }
 
+    /**
+     * Test override of default FileSystemProvider with the main application
+     * on the class path and a SecurityManager enabled.
+     */
+    public void testClassPathWithSecurityManager() throws Exception {
+        String moduleClasses = moduleClasses();
+        String testClasses = System.getProperty("test.classes");
+        String classpath = moduleClasses + File.pathSeparator + testClasses;
+        String policyFile = System.getProperty("test.src", ".")
+            + File.separator + "fs.policy";
+        int exitValue = exec(SET_DEFAULT_FSP, "-cp", classpath,
+            "-Dtest.classes=" + testClasses, "-Djava.security.manager",
+            "-Djava.security.policy==" + policyFile, "p.Main");
+        assertTrue(exitValue == 0);
+    }
+
     /**
      * Test override of default FileSystemProvider with the main application
      * on the module path as an exploded module.

test/jdk/java/nio/file/spi/fs.policy

@@ -0,0 +1,3 @@
+grant codeBase "file:${test.classes}${/}-" {
+    permission java.io.FilePermission "${java.io.tmpdir}${/}-", "write";
+};