8237041: AssertionError in parsing

Avoid parser crash for deeply nested classes without closing braces, improve error recovery for classes without an opening brace.

Reviewed-by: vromero

src/jdk.compiler/share/classes/com/sun/tools/javac/parser/JavacParser.java

@@ -444,7 +444,7 @@ public class JavacParser implements Parser {
             }
         }
         S.errPos(pos);
-        if (token.pos == errorPos) {
+        if (token.pos == errorPos && token.kind != EOF) {
             //check for a possible infinite loop in parsing:
             Assert.check(count++ < RECOVERY_THRESHOLD);
         } else {
@@ -4048,6 +4048,8 @@ public class JavacParser implements Parser {
             skip(false, true, false, false);
             if (token.kind == LBRACE)
                 nextToken();
+            else
+                return List.nil();
         }
         ListBuffer<JCTree> defs = new ListBuffer<>();
         while (token.kind != RBRACE && token.kind != EOF) {

test/langtools/tools/javac/parser/JavacParserTest.java

@@ -23,7 +23,7 @@
 
 /*
  * @test
- * @bug 7073631 7159445 7156633 8028235 8065753 8205418 8205913 8228451
+ * @bug 7073631 7159445 7156633 8028235 8065753 8205418 8205913 8228451 8237041
  * @summary tests error and diagnostics positions
  * @author  Jan Lahoda
  * @modules jdk.compiler/com.sun.tools.javac.api
@@ -1512,6 +1512,58 @@ public class JavacParserTest extends TestCase {
         }
     }
 
+    @Test //JDK-8237041
+    void testDeepNestingNoClose() throws IOException {
+        //verify that many nested unclosed classes do not crash javac
+        //due to the safety fallback in JavacParser.reportSyntaxError:
+        String code = "package t; class Test {\n";
+        for (int i = 0; i < 100; i++) {
+            code += "class C" + i + " {\n";
+        }
+        JavacTaskImpl ct = (JavacTaskImpl) tool.getTask(null, fm, null, List.of("-XDdev"),
+                null, Arrays.asList(new MyFileObject(code)));
+        Result result = ct.doCall();
+        assertEquals("Expected a (plain) error, got: " + result, result, Result.ERROR);
+    }
+
+    @Test //JDK-8237041
+    void testErrorRecoveryClassNotBrace() throws IOException {
+        //verify the AST form produced for classes without opening brace
+        //(classes without an opening brace do not nest the upcoming content):
+        String code = """
+                      package t;
+                      class Test {
+                          String.class,
+                          String.class,
+                          class A
+                          public
+                          class B
+                      }
+                      """;
+        JavacTaskImpl ct = (JavacTaskImpl) tool.getTask(null, fm, null, List.of("-XDdev"),
+                null, Arrays.asList(new MyFileObject(code)));
+        String ast = ct.parse().iterator().next().toString();
+        String expected = """
+                          package t;
+                          \n\
+                          class Test {
+                              String.<error> <error>;
+                              \n\
+                              class <error> {
+                              }
+                              \n\
+                              class <error> {
+                              }
+                              \n\
+                              class A {
+                              }
+                              \n\
+                              public class B {
+                              }
+                          }""";
+        assertEquals("Unexpected AST, got:\n" + ast, expected, ast);
+    }
+
     void run(String[] args) throws Exception {
         int passed = 0, failed = 0;
         final Pattern p = (args != null && args.length > 0)

test/langtools/tools/javac/processing/T6439826.java

@@ -60,9 +60,8 @@ public class T6439826 extends AbstractProcessor {
             System.err.print(s);
             // Expect the following 2 diagnostics, and no output to log
             //   Foo.java:1: illegal character: \35
-            //   Foo.java:1: reached end of file while parsing
             System.err.println(dl.count + " diagnostics; " + s.length() + " characters");
-            if (dl.count != 2 || s.length() != 0)
+            if (dl.count != 1 || s.length() != 0)
                 throw new AssertionError("unexpected output from compiler");
         }
     }

test/langtools/tools/javac/records/RecordCompilationTests.java

@@ -154,7 +154,7 @@ public class RecordCompilationTests extends CompilationTestCase {
 
     public void testMalformedDeclarations() {
         assertFail("compiler.err.premature.eof", "record R()");
-        assertFail("compiler.err.premature.eof", "record R();");
+        assertFail("compiler.err.expected", "record R();");
         assertFail("compiler.err.illegal.start.of.type", "record R(,) { }");
         assertFail("compiler.err.illegal.start.of.type", "record R((int x)) { }");
         assertFail("compiler.err.record.header.expected", "record R { }");