8223174: Pattern.compile() can throw confusing NegativeArraySizeException

Reviewed-by: martin

src/java.base/share/classes/java/util/regex/Pattern.java

@@ -1678,7 +1678,13 @@ loop:   for(int x=0, offset=0; x<nCodePoints; x++, offset+=len) {
             return;
         int j = i;
         i += 2;
-        int[] newtemp = new int[j + 3*(pLen-i) + 2];
+        int newTempLen;
+        try {
+            newTempLen = Math.addExact(j + 2, Math.multiplyExact(3, pLen - i));
+        } catch (ArithmeticException ae) {
+            throw new OutOfMemoryError();
+        }
+        int[] newtemp = new int[newTempLen];
         System.arraycopy(temp, 0, newtemp, 0, j);
 
         boolean inQuote = true;

test/jdk/java/util/regex/NegativeArraySize.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8223174
+ * @summary Pattern.compile() can throw confusing NegativeArraySizeException
+ * @requires os.maxMemory >= 5g
+ * @run main/othervm NegativeArraySize -Xms5G -Xmx5G
+ */
+
+import java.util.regex.Pattern;
+
+public class NegativeArraySize {
+    public static void main(String[] args) {
+        try {
+            Pattern.compile("\\Q" + "a".repeat(42 + Integer.MAX_VALUE / 3));
+            throw new AssertionError("expected to throw");
+        } catch (OutOfMemoryError expected) {
+        }
+    }
+}