8296684: Improve String platform support

Reviewed-by: amenkov, michaelm, rhalade

src/jdk.attach/aix/classes/sun/tools/attach/VirtualMachineImpl.java

@@ -136,6 +136,7 @@ public class VirtualMachineImpl extends HotSpotVirtualMachine {
      */
     InputStream execute(String cmd, Object ... args) throws AgentLoadException, IOException {
         assert args.length <= 3;                // includes null
+        checkNulls(args);
 
         // did we detach?
         synchronized (this) {

src/jdk.attach/linux/classes/sun/tools/attach/VirtualMachineImpl.java

@@ -140,6 +140,7 @@ public class VirtualMachineImpl extends HotSpotVirtualMachine {
      */
     InputStream execute(String cmd, Object ... args) throws AgentLoadException, IOException {
         assert args.length <= 3;                // includes null
+        checkNulls(args);
 
         // did we detach?
         synchronized (this) {

src/jdk.attach/macosx/classes/sun/tools/attach/VirtualMachineImpl.java

@@ -136,6 +136,7 @@ public class VirtualMachineImpl extends HotSpotVirtualMachine {
      */
     InputStream execute(String cmd, Object ... args) throws AgentLoadException, IOException {
         assert args.length <= 3;                // includes null
+        checkNulls(args);
 
         // did we detach?
         synchronized (this) {

src/jdk.attach/share/classes/sun/tools/attach/HotSpotVirtualMachine.java

@@ -487,4 +487,14 @@ public abstract class HotSpotVirtualMachine extends VirtualMachine {
         }
         return attachTimeout;
     }
+
+    protected static void checkNulls(Object... args) {
+        for (Object arg : args) {
+            if (arg instanceof String s) {
+                if (s.indexOf(0) >= 0) {
+                    throw new IllegalArgumentException("illegal null character in command");
+                }
+            }
+        }
+    }
 }

src/jdk.attach/windows/classes/sun/tools/attach/VirtualMachineImpl.java

@@ -73,6 +73,7 @@ public class VirtualMachineImpl extends HotSpotVirtualMachine {
         throws AgentLoadException, IOException
     {
         assert args.length <= 3;        // includes null
+        checkNulls(args);
 
         // create a pipe using a random name
         Random rnd = new Random();