8204290: Add check to limit number of capture groups

Reviewed-by: sundar, jlaskey

src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/regexp/joni/Config.java

@@ -45,6 +45,7 @@ public interface Config {
 
     final int NREGION                   = 10;
     final int MAX_BACKREF_NUM           = 1000;
+    final int MAX_CAPTURE_GROUP_NUM     = 0x8000;
     final int MAX_REPEAT_NUM            = 100000;
     final int MAX_MULTI_BYTE_RANGES_NUM = 10000;
 

src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/regexp/joni/ScanEnvironment.java

@@ -62,6 +62,9 @@ public final class ScanEnvironment {
     }
 
     public int addMemEntry() {
+        if (numMem >= Config.MAX_CAPTURE_GROUP_NUM) {
+            throw new InternalException(ErrorMessages.ERR_TOO_MANY_CAPTURE_GROUPS);
+        }
         if (numMem++ == 0) {
             memNodes = new Node[SCANENV_MEMNODES_SIZE];
         } else if (numMem >= memNodes.length) {

src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/regexp/joni/exception/ErrorMessages.java

@@ -31,6 +31,7 @@ public interface ErrorMessages {
     final String ERR_PARSER_BUG = "internal parser error (bug)";
     final String ERR_UNDEFINED_BYTECODE = "undefined bytecode (bug)";
     final String ERR_UNEXPECTED_BYTECODE = "unexpected bytecode (bug)";
+    final String ERR_TOO_MANY_CAPTURE_GROUPS = "too many capture groups";
 
     /* syntax error */
     final String ERR_END_PATTERN_AT_LEFT_BRACE = "end pattern at left brace";

test/nashorn/script/basic/JDK-8204290.js

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ * 
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ * 
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ * 
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * 
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * JDK-8204290: Add check to limit number of capture groups
+ *
+ * @test
+ * @run
+ */
+
+try {
+    new RegExp("()".repeat(0x8001));
+    fail("Expected exception");
+} catch (e) {
+    Assert.assertTrue(e instanceof SyntaxError);
+    Assert.assertEquals(e.message, "too many capture groups");
+}
+