8166258: Unexpected code conversion by HKSCS converters

Reviewed-by: naoto

jdk/src/java.base/share/classes/sun/nio/cs/HKSCS.java

@@ -432,6 +432,8 @@ public class HKSCS {
                     continue;
                 for (int i = 0; i < s.length(); i++) {
                     char c = s.charAt(i);
+                    if (c == UNMAPPABLE_DECODING)
+                        continue;
                     int hi = c >> 8;
                     if (c2b[hi] == C2B_UNMAPPABLE) {
                         c2b[hi] = new char[0x100];

jdk/test/sun/nio/cs/TestHKSCS.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8166258
+ * @summary Some corner cases for hkscs charsets
+ * @modules jdk.charsets
+ * @run main TestHKSCS
+ */
+
+import java.util.Arrays;
+
+public class TestHKSCS {
+    public static void main(String args[]) throws Exception {
+        String[] charsets = { "x-MS950-HKSCS-XP",
+                              "x-MS950-HKSCS",
+                              "Big5-HKSCS",
+                              "x-Big5-HKSCS-2001"
+        };
+        String s = "\ufffd\ud87f\udffd";
+        byte[] bytes = new byte[] { 0x3f, 0x3f };
+        for (String cs : charsets) {
+            if (!Arrays.equals(bytes, s.getBytes(cs))) {
+                throw new RuntimeException(cs + " failed to decode u+fffd");
+            }
+        }
+    }
+}