8024512: Regex /[^\[]/ doesn't match

Reviewed-by: jlaskey, sundar

nashorn/src/jdk/nashorn/internal/runtime/regexp/RegExpScanner.java

@@ -263,15 +263,6 @@ final class RegExpScanner extends Scanner {
         }
 
         if (atom()) {
-            // Check for character classes that never or always match
-            if (sb.toString().endsWith("[]")) {
-                sb.setLength(sb.length() - 1);
-                sb.append("^\\s\\S]");
-            } else if (sb.toString().endsWith("[^]")) {
-                sb.setLength(sb.length() - 2);
-                sb.append("\\s\\S]");
-            }
-
             quantifier();
             return true;
         }
@@ -767,7 +758,18 @@ final class RegExpScanner extends Scanner {
 
                 if (classRanges() && ch0 == ']') {
                     pop(']');
-                    return commit(1);
+                    commit(1);
+
+                    // Substitute empty character classes [] and [^] that never or always match
+                    if (position == startIn + 2) {
+                        sb.setLength(sb.length() - 1);
+                        sb.append("^\\s\\S]");
+                    } else if (position == startIn + 3 && inNegativeClass) {
+                        sb.setLength(sb.length() - 2);
+                        sb.append("\\s\\S]");
+                    }
+
+                    return true;
                 }
             } finally {
                 inCharClass = false;  // no nested character classes in JavaScript

nashorn/test/script/basic/JDK-8024512.js

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ * 
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ * 
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ * 
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * 
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * JDK-8024512: Regex /[^\[]/ doesn't match
+ *
+ * @test
+ * @run
+ */
+
+print("[M]".match(/(\[[^\[]*\])/));
+print("[[]".match(/(\[[^\[]*\])/));
+
+print("[M]".match(/(\[[^\[^]*\])/));
+print("[[]".match(/(\[[^\[^]*\])/));
+print("[^]".match(/(\[[^\[^]*\])/));
+
+print("[M]".match(/(\[[^\[]\])/));
+print("[[]".match(/(\[[^\[]\])/));
+
+print("[M]".match(/(\[[^\[^]\])/));
+print("[[]".match(/(\[[^\[^]\])/));
+print("[^]".match(/(\[[^\[^]\])/));
+
+print("M".match(/[^\[]/));
+print("[".match(/[^\[]/));
+print("^".match(/[^\[]/));
+
+// Repeat above without escaping inner square bracket
+print("[M]".match(/(\[[^[]\])/));
+print("[[]".match(/(\[[^[]\])/));
+
+print("[M]".match(/(\[[^[^]\])/));
+print("[[]".match(/(\[[^[^]\])/));
+print("[^]".match(/(\[[^[^]\])/));
+
+print("M".match(/[^[]/));
+print("[".match(/[^[]/));
+print("^".match(/[^[]/));

nashorn/test/script/basic/JDK-8024512.js.EXPECTED

@@ -0,0 +1,21 @@
+[M],[M]
+[],[]
+[M],[M]
+[],[]
+null
+[M],[M]
+null
+[M],[M]
+null
+null
+M
+null
+^
+[M],[M]
+null
+[M],[M]
+null
+null
+M
+null
+^