8261969: SNIHostName should check if the encoded hostname conform to RFC 3490

Reviewed-by: rhalade, xuelei

src/java.base/share/classes/javax/net/ssl/SNIHostName.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -173,7 +173,8 @@ public final class SNIHostName extends SNIServerName {
                     .onUnmappableCharacter(CodingErrorAction.REPORT);
 
             this.hostname = IDN.toASCII(
-                    decoder.decode(ByteBuffer.wrap(encoded)).toString());
+                    decoder.decode(ByteBuffer.wrap(encoded)).toString(),
+                    IDN.USE_STD3_ASCII_RULES);
         } catch (RuntimeException | CharacterCodingException e) {
             throw new IllegalArgumentException(
                         "The encoded server name value is invalid", e);

test/jdk/javax/net/ssl/ServerName/IllegalSNIName.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,32 +23,50 @@
 
 /*
  * @test
- * @bug 8020842
- * @summary SNIHostName does not throw IAE when hostname ends
- *          with a trailing dot
+ * @bug 8020842 8261969
+ * @summary SNIHostName does not throw IAE when hostname doesn't conform to
+ *          RFC 3490 or ends with a trailing dot
  */
 
 import javax.net.ssl.SNIHostName;
+import java.nio.charset.StandardCharsets;
+import java.util.HexFormat;
 
 public class IllegalSNIName {
 
+    private static void checkHostname(String hostname) throws Exception {
+        try {
+            new SNIHostName(hostname);
+            throw new RuntimeException("Expected to get IllegalArgumentException for "
+                    + hostname);
+        } catch (IllegalArgumentException iae) {
+            // That's the right behavior.
+        }
+    }
+
+    private static void checkHostname(byte[] encodedHostname) throws Exception {
+        try {
+            new SNIHostName(encodedHostname);
+            throw new RuntimeException("Expected to get IllegalArgumentException for "
+                    + HexFormat.ofDelimiter(":").formatHex(encodedHostname));
+        } catch (IllegalArgumentException iae) {
+            // That's the right behavior.
+        }
+    }
+
     public static void main(String[] args) throws Exception {
         String[] illegalNames = {
                 "example\u3002\u3002com",
                 "example..com",
                 "com\u3002",
                 "com.",
-                "."
-            };
+                ".",
+                "example^com"
+        };
 
         for (String name : illegalNames) {
-            try {
-                SNIHostName hostname = new SNIHostName(name);
-                throw new Exception(
-                    "Expected to get IllegalArgumentException for " + name);
-            } catch (IllegalArgumentException iae) {
-                // That's the right behavior.
-            }
+            checkHostname(name);
+            checkHostname(name.getBytes(StandardCharsets.UTF_8));
         }
     }
 }