8078320: Improve DocTrees parsing

Reviewed-by: jjg, jlahoda

langtools/src/jdk.compiler/share/classes/com/sun/source/doctree/DocCommentTree.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
 
 package com.sun.source.doctree;
 
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -43,6 +44,20 @@ public interface DocCommentTree extends DocTree {
      */
     List<? extends DocTree> getFirstSentence();
 
+    /**
+     * Returns the entire body of a documentation comment, appearing
+     * before any block tags, including the first sentence.
+     * @return body of a documentation comment first sentence inclusive
+     *
+     * @since 1.9
+     */
+    default List<? extends DocTree> getFullBody() {
+        ArrayList<DocTree> bodyList = new ArrayList<>();
+        bodyList.addAll(getFirstSentence());
+        bodyList.addAll(getBody());
+        return bodyList;
+    }
+
     /**
      * Returns the body of a documentation comment,
      * appearing after the first sentence, and before any block tags.

langtools/src/jdk.compiler/share/classes/com/sun/source/util/DocTrees.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,12 +25,15 @@
 
 package com.sun.source.util;
 
+import java.util.List;
+
 import javax.annotation.processing.ProcessingEnvironment;
 import javax.lang.model.element.Element;
+import javax.tools.Diagnostic;
 import javax.tools.JavaCompiler.CompilationTask;
 
 import com.sun.source.doctree.DocCommentTree;
-import javax.tools.Diagnostic;
+import com.sun.source.doctree.DocTree;
 
 /**
  * Provides access to syntax trees for doc comments.
@@ -77,6 +80,17 @@ public abstract class DocTrees extends Trees {
      */
     public abstract Element getElement(DocTreePath path);
 
+    /**
+     * Returns the list of {@link DocTree} representing the first sentence of
+     * a comment.
+     *
+     * @param list the DocTree list to interrogate
+     * @return the first sentence
+     *
+     * @since 1.9
+     */
+    public abstract List<DocTree> getFirstSentence(List<? extends DocTree> list);
+
     /**
      * Returns a utility object for accessing the source positions
      * of documentation tree nodes.

langtools/src/jdk.compiler/share/classes/com/sun/tools/doclint/HtmlTag.java

@@ -25,19 +25,18 @@
 
 package com.sun.tools.doclint;
 
-import java.util.Set;
 import java.util.Collections;
 import java.util.EnumMap;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
-
+import java.util.Set;
 import javax.lang.model.element.Name;
 
-import static com.sun.tools.doclint.HtmlTag.Attr.*;
-
 import com.sun.tools.javac.util.StringUtils;
 
+import static com.sun.tools.doclint.HtmlTag.Attr.*;
+
 /**
  * Enum representing HTML tags.
  *
@@ -646,15 +645,14 @@ public enum HtmlTag {
         return map;
     }
 
-    private static final Map<String,HtmlTag> index = new HashMap<>();
+    private static final Map<String, HtmlTag> index = new HashMap<>();
     static {
         for (HtmlTag t: values()) {
             index.put(t.getText(), t);
         }
     }
 
-    static HtmlTag get(Name tagName) {
+    public static HtmlTag get(Name tagName) {
         return index.get(StringUtils.toLowerCase(tagName.toString()));
     }
-
 }

langtools/src/jdk.compiler/share/classes/com/sun/tools/javac/api/JavacTrees.java

@@ -25,7 +25,6 @@
 
 package com.sun.tools.javac.api;
 
-import java.io.IOException;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -86,9 +85,17 @@ import com.sun.tools.javac.tree.DCTree.DCIdentifier;
 import com.sun.tools.javac.tree.DCTree.DCParam;
 import com.sun.tools.javac.tree.DCTree.DCReference;
 import com.sun.tools.javac.tree.DCTree.DCText;
+import com.sun.tools.javac.tree.DocTreeMaker;
 import com.sun.tools.javac.tree.EndPosTable;
 import com.sun.tools.javac.tree.JCTree;
-import com.sun.tools.javac.tree.JCTree.*;
+import com.sun.tools.javac.tree.JCTree.JCBlock;
+import com.sun.tools.javac.tree.JCTree.JCCatch;
+import com.sun.tools.javac.tree.JCTree.JCClassDecl;
+import com.sun.tools.javac.tree.JCTree.JCCompilationUnit;
+import com.sun.tools.javac.tree.JCTree.JCExpression;
+import com.sun.tools.javac.tree.JCTree.JCIdent;
+import com.sun.tools.javac.tree.JCTree.JCMethodDecl;
+import com.sun.tools.javac.tree.JCTree.JCVariableDecl;
 import com.sun.tools.javac.tree.TreeCopier;
 import com.sun.tools.javac.tree.TreeInfo;
 import com.sun.tools.javac.tree.TreeMaker;
@@ -106,6 +113,7 @@ import com.sun.tools.javac.util.Name;
 import com.sun.tools.javac.util.Names;
 import com.sun.tools.javac.util.Pair;
 import com.sun.tools.javac.util.Position;
+
 import static com.sun.tools.javac.code.Kinds.Kind.*;
 import static com.sun.tools.javac.code.TypeTag.*;
 
@@ -132,6 +140,7 @@ public class JavacTrees extends DocTrees {
     private JavacTaskImpl javacTaskImpl;
     private Names names;
     private Types types;
+    private DocTreeMaker doctreeMaker;
 
     // called reflectively from Trees.instance(CompilationTask task)
     public static JavacTrees instance(JavaCompiler.CompilationTask task) {
@@ -173,6 +182,7 @@ public class JavacTrees extends DocTrees {
         memberEnter = MemberEnter.instance(context);
         names = Names.instance(context);
         types = Types.instance(context);
+        doctreeMaker = DocTreeMaker.instance(context);
 
         JavacTask t = context.get(JavacTask.class);
         if (t instanceof JavacTaskImpl)
@@ -259,7 +269,7 @@ public class JavacTrees extends DocTrees {
 
         tree.accept(new DocTreeScanner<Void, Void>() {
             @Override @DefinedBy(Api.COMPILER_TREE)
- public Void scan(DocTree node, Void p) {
+            public Void scan(DocTree node, Void p) {
                 if (node != null) last[0] = node;
                 return null;
             }
@@ -356,6 +366,11 @@ public class JavacTrees extends DocTrees {
         return null;
     }
 
+    @Override @DefinedBy(Api.COMPILER_TREE)
+    public java.util.List<DocTree> getFirstSentence(java.util.List<? extends DocTree> list) {
+        return doctreeMaker.getFirstSentence(list);
+    }
+
     private Symbol attributeDocReference(TreePath path, DCReference ref) {
         Env<AttrContext> env = getAttrContext(path);
 
@@ -763,7 +778,6 @@ public class JavacTrees extends DocTrees {
             javacTaskImpl.enter(null);
         }
 
-
         JCCompilationUnit unit = (JCCompilationUnit) path.getCompilationUnit();
         Copier copier = createCopier(treeMaker.forToplevel(unit));
 

langtools/src/jdk.compiler/share/classes/com/sun/tools/javac/parser/DocCommentParser.java

@@ -26,12 +26,8 @@
 package com.sun.tools.javac.parser;
 
 import java.text.BreakIterator;
-import java.util.Arrays;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
 
 import com.sun.source.doctree.AttributeTree.ValueKind;
 import com.sun.tools.javac.parser.DocCommentParser.TagParser.Kind;
@@ -40,12 +36,10 @@ import com.sun.tools.javac.parser.Tokens.TokenKind;
 import com.sun.tools.javac.tree.DCTree;
 import com.sun.tools.javac.tree.DCTree.DCAttribute;
 import com.sun.tools.javac.tree.DCTree.DCDocComment;
-import com.sun.tools.javac.tree.DCTree.DCEndElement;
 import com.sun.tools.javac.tree.DCTree.DCEndPosTree;
 import com.sun.tools.javac.tree.DCTree.DCErroneous;
 import com.sun.tools.javac.tree.DCTree.DCIdentifier;
 import com.sun.tools.javac.tree.DCTree.DCReference;
-import com.sun.tools.javac.tree.DCTree.DCStartElement;
 import com.sun.tools.javac.tree.DCTree.DCText;
 import com.sun.tools.javac.tree.DocTreeMaker;
 import com.sun.tools.javac.tree.JCTree;
@@ -55,9 +49,8 @@ import com.sun.tools.javac.util.ListBuffer;
 import com.sun.tools.javac.util.Log;
 import com.sun.tools.javac.util.Name;
 import com.sun.tools.javac.util.Names;
-import com.sun.tools.javac.util.Options;
 import com.sun.tools.javac.util.Position;
-import com.sun.tools.javac.util.StringUtils;
+
 import static com.sun.tools.javac.util.LayoutCharacters.*;
 
 /**
@@ -100,24 +93,20 @@ public class DocCommentParser {
 
     Map<Name, TagParser> tagParsers;
 
-    DocCommentParser(ParserFactory fac, DiagnosticSource diagSource, Comment comment) {
+    public DocCommentParser(ParserFactory fac, DiagnosticSource diagSource, Comment comment) {
         this.fac = fac;
         this.diagSource = diagSource;
         this.comment = comment;
         names = fac.names;
         m = fac.docTreeMaker;
-
-        Locale locale = (fac.locale == null) ? Locale.getDefault() : fac.locale;
-
-        Options options = fac.options;
-        boolean useBreakIterator = options.isSet("breakIterator");
-        if (useBreakIterator || !locale.getLanguage().equals(Locale.ENGLISH.getLanguage()))
-            sentenceBreaker = BreakIterator.getSentenceInstance(locale);
-
         initTagParsers();
     }
 
-    DCDocComment parse() {
+    public DocCommentParser(ParserFactory fac) {
+        this(fac, null, null);
+    }
+
+    public DCDocComment parse() {
         String c = comment.getText();
         buf = new char[c.length() + 1];
         c.getChars(0, c.length(), buf, 0);
@@ -128,54 +117,11 @@ public class DocCommentParser {
 
         List<DCTree> body = blockContent();
         List<DCTree> tags = blockTags();
+        int pos = !body.isEmpty()
+                ? body.head.pos
+                : !tags.isEmpty() ? tags.head.pos : Position.NOPOS;
 
-        // split body into first sentence and body
+        DCDocComment dc = m.at(pos).DocComment(comment, body, tags);
-        ListBuffer<DCTree> fs = new ListBuffer<>();
-        loop:
-        for (; body.nonEmpty(); body = body.tail) {
-            DCTree t = body.head;
-            switch (t.getKind()) {
-                case TEXT:
-                    String s = ((DCText) t).getBody();
-                    int i = getSentenceBreak(s);
-                    if (i > 0) {
-                        int i0 = i;
-                        while (i0 > 0 && isWhitespace(s.charAt(i0 - 1)))
-                            i0--;
-                        fs.add(m.at(t.pos).Text(s.substring(0, i0)));
-                        int i1 = i;
-                        while (i1 < s.length() && isWhitespace(s.charAt(i1)))
-                            i1++;
-                        body = body.tail;
-                        if (i1 < s.length())
-                            body = body.prepend(m.at(t.pos + i1).Text(s.substring(i1)));
-                        break loop;
-                    } else if (body.tail.nonEmpty()) {
-                        if (isSentenceBreak(body.tail.head)) {
-                            int i0 = s.length() - 1;
-                            while (i0 > 0 && isWhitespace(s.charAt(i0)))
-                                i0--;
-                            fs.add(m.at(t.pos).Text(s.substring(0, i0 + 1)));
-                            body = body.tail;
-                            break loop;
-                        }
-                    }
-                    break;
-
-                case START_ELEMENT:
-                case END_ELEMENT:
-                    if (isSentenceBreak(t))
-                        break loop;
-                    break;
-            }
-            fs.add(t);
-        }
-
-        @SuppressWarnings("unchecked")
-        DCTree first = getFirst(fs.toList(), body, tags);
-        int pos = (first == null) ? Position.NOPOS : first.pos;
-
-        DCDocComment dc = m.at(pos).DocComment(comment, fs.toList(), body, tags);
         return dc;
     }
 
@@ -331,23 +277,28 @@ public class DocCommentParser {
             nextChar();
             if (isIdentifierStart(ch)) {
                 Name name = readTagName();
-                skipWhitespace();
-
                 TagParser tp = tagParsers.get(name);
+
                 if (tp == null) {
-                    DCTree text = inlineText();
+                    skipWhitespace();
+                    DCTree text = inlineText(WhitespaceRetentionPolicy.REMOVE_ALL);
                     if (text != null) {
                         nextChar();
                         return m.at(p).UnknownInlineTag(name, List.of(text)).setEndPos(bp);
                     }
-                } else if (tp.getKind() == TagParser.Kind.INLINE) {
-                    DCEndPosTree<?> tree = (DCEndPosTree<?>) tp.parse(p);
-                    if (tree != null) {
-                        return tree.setEndPos(bp);
-                    }
                 } else {
-                    inlineText(); // skip content
+                    if (!tp.retainWhiteSpace) {
-                    nextChar();
+                        skipWhitespace();
+                    }
+                    if (tp.getKind() == TagParser.Kind.INLINE) {
+                        DCEndPosTree<?> tree = (DCEndPosTree<?>) tp.parse(p);
+                        if (tree != null) {
+                            return tree.setEndPos(bp);
+                        }
+                    } else { // handle block tags (ex: @see) in inline content
+                        inlineText(WhitespaceRetentionPolicy.REMOVE_ALL); // skip content
+                        nextChar();
+                    }
                 }
             }
             return erroneous("dc.no.tag.name", p);
@@ -356,13 +307,32 @@ public class DocCommentParser {
         }
     }
 
+    private static enum WhitespaceRetentionPolicy {
+        RETAIN_ALL,
+        REMOVE_FIRST_SPACE,
+        REMOVE_ALL
+    }
+
     /**
      * Read plain text content of an inline tag.
      * Matching pairs of { } are skipped; the text is terminated by the first
      * unmatched }. It is an error if the beginning of the next tag is detected.
      */
-    protected DCTree inlineText() throws ParseException {
+    private DCTree inlineText(WhitespaceRetentionPolicy whitespacePolicy) throws ParseException {
-        skipWhitespace();
+        switch (whitespacePolicy) {
+            case REMOVE_ALL:
+                skipWhitespace();
+                break;
+            case REMOVE_FIRST_SPACE:
+                if (ch == ' ')
+                    nextChar();
+                break;
+            case RETAIN_ALL:
+            default:
+                // do nothing
+                break;
+
+        }
         int pos = bp;
         int depth = 1;
 
@@ -742,7 +712,8 @@ public class DocCommentParser {
                 }
                 if (ch == '>') {
                     nextChar();
-                    return m.at(p).StartElement(name, attrs, selfClosing).setEndPos(bp);
+                    DCTree dctree = m.at(p).StartElement(name, attrs, selfClosing).setEndPos(bp);
+                    return dctree;
                 }
             }
         } else if (ch == '/') {
@@ -884,15 +855,6 @@ public class DocCommentParser {
         return m.at(pos).Erroneous(newString(pos, i + 1), diagSource, code);
     }
 
-    @SuppressWarnings("unchecked")
-    <T> T getFirst(List<T>... lists) {
-        for (List<T> list: lists) {
-            if (list.nonEmpty())
-                return list.head;
-        }
-        return null;
-    }
-
     protected boolean isIdentifierStart(char ch) {
         return Character.isUnicodeIdentifierStart(ch);
     }
@@ -916,8 +878,11 @@ public class DocCommentParser {
     protected Name readTagName() {
         int start = bp;
         nextChar();
-        while (bp < buflen && (Character.isUnicodeIdentifierPart(ch) || ch == '.'))
+        while (bp < buflen
+                && (Character.isUnicodeIdentifierPart(ch) || ch == '.'
+                || ch == '-' || ch == ':')) {
             nextChar();
+        }
         return names.fromChars(buf, start, bp - start);
     }
 
@@ -960,59 +925,9 @@ public class DocCommentParser {
     }
 
     protected void skipWhitespace() {
-        while (isWhitespace(ch))
+        while (isWhitespace(ch)) {
             nextChar();
-    }
-
-    protected int getSentenceBreak(String s) {
-        if (sentenceBreaker != null) {
-            sentenceBreaker.setText(s);
-            int i = sentenceBreaker.next();
-            return (i == s.length()) ? -1 : i;
         }
-
-        // scan for period followed by whitespace
-        boolean period = false;
-        for (int i = 0; i < s.length(); i++) {
-            switch (s.charAt(i)) {
-                case '.':
-                    period = true;
-                    break;
-
-                case ' ':
-                case '\f':
-                case '\n':
-                case '\r':
-                case '\t':
-                    if (period)
-                        return i;
-                    break;
-
-                default:
-                    period = false;
-                    break;
-            }
-        }
-        return -1;
-    }
-
-
-    Set<String> htmlBlockTags = new HashSet<>(Arrays.asList(
-                    "h1", "h2", "h3", "h4", "h5", "h6", "p", "pre"));
-
-    protected boolean isSentenceBreak(Name n) {
-        return htmlBlockTags.contains(StringUtils.toLowerCase(n.toString()));
-    }
-
-    protected boolean isSentenceBreak(DCTree t) {
-        switch (t.getKind()) {
-            case START_ELEMENT:
-                return isSentenceBreak(((DCStartElement) t).getName());
-
-            case END_ELEMENT:
-                return isSentenceBreak(((DCEndElement) t).getName());
-        }
-        return false;
     }
 
     /**
@@ -1026,12 +941,21 @@ public class DocCommentParser {
     static abstract class TagParser {
         enum Kind { INLINE, BLOCK }
 
-        Kind kind;
+        final Kind kind;
-        DCTree.Kind treeKind;
+        final DCTree.Kind treeKind;
+        final boolean retainWhiteSpace;
+
 
         TagParser(Kind k, DCTree.Kind tk) {
             kind = k;
             treeKind = tk;
+            retainWhiteSpace = false;
+        }
+
+        TagParser(Kind k, DCTree.Kind tk, boolean retainWhiteSpace) {
+            kind = k;
+            treeKind = tk;
+            this.retainWhiteSpace = retainWhiteSpace;
         }
 
         Kind getKind() {
@@ -1059,9 +983,9 @@ public class DocCommentParser {
             },
 
             // {@code text}
-            new TagParser(Kind.INLINE, DCTree.Kind.CODE) {
+            new TagParser(Kind.INLINE, DCTree.Kind.CODE, true) {
                 public DCTree parse(int pos) throws ParseException {
-                    DCTree text = inlineText();
+                    DCTree text = inlineText(WhitespaceRetentionPolicy.REMOVE_FIRST_SPACE);
                     nextChar();
                     return m.at(pos).Code((DCText) text);
                 }
@@ -1082,7 +1006,7 @@ public class DocCommentParser {
                         nextChar();
                         return m.at(pos).DocRoot();
                     }
-                    inlineText(); // skip unexpected content
+                    inlineText(WhitespaceRetentionPolicy.REMOVE_ALL); // skip unexpected content
                     nextChar();
                     throw new ParseException("dc.unexpected.content");
                 }
@@ -1105,7 +1029,7 @@ public class DocCommentParser {
                         nextChar();
                         return m.at(pos).InheritDoc();
                     }
-                    inlineText(); // skip unexpected content
+                    inlineText(WhitespaceRetentionPolicy.REMOVE_ALL); // skip unexpected content
                     nextChar();
                     throw new ParseException("dc.unexpected.content");
                 }
@@ -1130,9 +1054,9 @@ public class DocCommentParser {
             },
 
             // {@literal text}
-            new TagParser(Kind.INLINE, DCTree.Kind.LITERAL) {
+            new TagParser(Kind.INLINE, DCTree.Kind.LITERAL, true) {
                 public DCTree parse(int pos) throws ParseException {
-                    DCTree text = inlineText();
+                    DCTree text = inlineText(WhitespaceRetentionPolicy.REMOVE_FIRST_SPACE);
                     nextChar();
                     return m.at(pos).Literal((DCText) text);
                 }

langtools/src/jdk.compiler/share/classes/com/sun/tools/javac/tree/DCTree.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,6 @@
 
 package com.sun.tools.javac.tree;
 
-
 import javax.tools.Diagnostic;
 
 import com.sun.source.doctree.*;
@@ -39,8 +38,10 @@ import com.sun.tools.javac.util.JCDiagnostic.SimpleDiagnosticPosition;
 import com.sun.tools.javac.util.List;
 import com.sun.tools.javac.util.Name;
 import com.sun.tools.javac.util.Position;
+
 import java.io.IOException;
 import java.io.StringWriter;
+
 import javax.tools.JavaFileObject;
 
 /**
@@ -104,14 +105,19 @@ public abstract class DCTree implements DocTree {
     public static class DCDocComment extends DCTree implements DocCommentTree {
         public final Comment comment; // required for the implicit source pos table
 
+        public final List<DCTree> fullBody;
         public final List<DCTree> firstSentence;
         public final List<DCTree> body;
         public final List<DCTree> tags;
 
         public DCDocComment(Comment comment,
-                List<DCTree> firstSentence, List<DCTree> body, List<DCTree> tags) {
+                            List<DCTree> fullBody,
+                            List<DCTree> firstSentence,
+                            List<DCTree> body,
+                            List<DCTree> tags) {
             this.comment = comment;
             this.firstSentence = firstSentence;
+            this.fullBody = fullBody;
             this.body = body;
             this.tags = tags;
         }
@@ -131,6 +137,11 @@ public abstract class DCTree implements DocTree {
             return firstSentence;
         }
 
+        @DefinedBy(Api.COMPILER_TREE)
+        public List<? extends DocTree> getFullBody() {
+            return fullBody;
+        }
+
         @DefinedBy(Api.COMPILER_TREE)
         public List<? extends DocTree> getBody() {
             return body;

langtools/src/jdk.compiler/share/classes/com/sun/tools/javac/tree/DocPretty.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,15 +25,15 @@
 
 package com.sun.tools.javac.tree;
 
+import java.io.IOException;
 import java.io.Writer;
+import java.util.List;
 
 import com.sun.source.doctree.*;
 import com.sun.source.doctree.AttributeTree.ValueKind;
 import com.sun.tools.javac.util.Convert;
 import com.sun.tools.javac.util.DefinedBy;
 import com.sun.tools.javac.util.DefinedBy.Api;
-import java.io.IOException;
-import java.util.List;
 
 /**
  * Prints out a doc comment tree.
@@ -201,14 +201,10 @@ public class DocPretty implements DocTreeVisitor<Void,Void> {
     @DefinedBy(Api.COMPILER_TREE)
     public Void visitDocComment(DocCommentTree node, Void p) {
         try {
-            List<? extends DocTree> fs = node.getFirstSentence();
+            List<? extends DocTree> b = node.getFullBody();
-            List<? extends DocTree> b = node.getBody();
             List<? extends DocTree> t = node.getBlockTags();
-            print(fs);
-            if (!fs.isEmpty() && !b.isEmpty())
-                print(" ");
             print(b);
-            if ((!fs.isEmpty() || !b.isEmpty()) && !t.isEmpty())
+            if (!b.isEmpty() && !t.isEmpty())
                 print("\n");
             print(t, "\n");
         } catch (IOException e) {
@@ -308,7 +304,10 @@ public class DocPretty implements DocTreeVisitor<Void,Void> {
         try {
             print("{");
             printTagName(node);
-            print(" ");
+            String body = node.getBody().getBody();
+            if (!body.isEmpty() && !Character.isWhitespace(body.charAt(0))) {
+                print(" ");
+            }
             print(node.getBody());
             print("}");
         } catch (IOException e) {

langtools/src/jdk.compiler/share/classes/com/sun/tools/javac/tree/DocTreeMaker.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,19 +25,62 @@
 
 package com.sun.tools.javac.tree;
 
+import java.text.BreakIterator;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.ListIterator;
+import java.util.Locale;
+
 import com.sun.source.doctree.AttributeTree.ValueKind;
+import com.sun.source.doctree.DocTree;
 import com.sun.source.doctree.DocTree.Kind;
+import com.sun.source.doctree.EndElementTree;
+import com.sun.source.doctree.StartElementTree;
+import com.sun.tools.doclint.HtmlTag;
 
 import com.sun.tools.javac.parser.Tokens.Comment;
-import com.sun.tools.javac.tree.DCTree.*;
+import com.sun.tools.javac.tree.DCTree.DCAttribute;
+import com.sun.tools.javac.tree.DCTree.DCAuthor;
+import com.sun.tools.javac.tree.DCTree.DCComment;
+import com.sun.tools.javac.tree.DCTree.DCDeprecated;
+import com.sun.tools.javac.tree.DCTree.DCDocComment;
+import com.sun.tools.javac.tree.DCTree.DCDocRoot;
+import com.sun.tools.javac.tree.DCTree.DCEndElement;
+import com.sun.tools.javac.tree.DCTree.DCEntity;
+import com.sun.tools.javac.tree.DCTree.DCErroneous;
+import com.sun.tools.javac.tree.DCTree.DCIdentifier;
+import com.sun.tools.javac.tree.DCTree.DCInheritDoc;
+import com.sun.tools.javac.tree.DCTree.DCLink;
+import com.sun.tools.javac.tree.DCTree.DCLiteral;
+import com.sun.tools.javac.tree.DCTree.DCParam;
+import com.sun.tools.javac.tree.DCTree.DCReference;
+import com.sun.tools.javac.tree.DCTree.DCReturn;
+import com.sun.tools.javac.tree.DCTree.DCSee;
+import com.sun.tools.javac.tree.DCTree.DCSerial;
+import com.sun.tools.javac.tree.DCTree.DCSerialData;
+import com.sun.tools.javac.tree.DCTree.DCSerialField;
+import com.sun.tools.javac.tree.DCTree.DCSince;
+import com.sun.tools.javac.tree.DCTree.DCStartElement;
+import com.sun.tools.javac.tree.DCTree.DCText;
+import com.sun.tools.javac.tree.DCTree.DCThrows;
+import com.sun.tools.javac.tree.DCTree.DCUnknownBlockTag;
+import com.sun.tools.javac.tree.DCTree.DCUnknownInlineTag;
+import com.sun.tools.javac.tree.DCTree.DCValue;
+import com.sun.tools.javac.tree.DCTree.DCVersion;
 import com.sun.tools.javac.util.Context;
 import com.sun.tools.javac.util.DiagnosticSource;
 import com.sun.tools.javac.util.JCDiagnostic;
 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticPosition;
 import com.sun.tools.javac.util.List;
+import com.sun.tools.javac.util.ListBuffer;
 import com.sun.tools.javac.util.Name;
+import com.sun.tools.javac.util.Options;
+import com.sun.tools.javac.util.Pair;
 import com.sun.tools.javac.util.Position;
 
+import static com.sun.tools.doclint.HtmlTag.*;
+
 /**
  *
  *  <p><b>This is NOT part of any supported API.
@@ -50,6 +93,12 @@ public class DocTreeMaker {
     /** The context key for the tree factory. */
     protected static final Context.Key<DocTreeMaker> treeMakerKey = new Context.Key<>();
 
+    // A subset of block tags, which acts as sentence breakers, appearing
+    // anywhere but the zero'th position in the first sentence.
+    final EnumSet<HtmlTag> sentenceBreakTags;
+
+    private final BreakIterator sentenceBreaker;
+
     /** Get the TreeMaker instance. */
     public static DocTreeMaker instance(Context context) {
         DocTreeMaker instance = context.get(treeMakerKey);
@@ -71,6 +120,15 @@ public class DocTreeMaker {
         context.put(treeMakerKey, this);
         diags = JCDiagnostic.Factory.instance(context);
         this.pos = Position.NOPOS;
+        sentenceBreakTags = EnumSet.of(H1, H2, H3, H4, H5, H6, PRE, P);
+        Locale locale = (context.get(Locale.class) != null)
+                ? context.get(Locale.class)
+                : Locale.getDefault();
+        Options options = Options.instance(context);
+        boolean useBreakIterator = options.isSet("breakiterator");
+        sentenceBreaker = (useBreakIterator || !locale.getLanguage().equals(Locale.ENGLISH.getLanguage()))
+                ? BreakIterator.getSentenceInstance(locale)
+                : null;
     }
 
     /** Reassign current position.
@@ -117,9 +175,11 @@ public class DocTreeMaker {
         return tree;
     }
 
-    public DCDocComment DocComment(Comment comment, List<DCTree> firstSentence, List<DCTree> body, List<DCTree> tags) {
+    public DCDocComment DocComment(Comment comment, List<DCTree> fullBody, List<DCTree> tags) {
-        DCDocComment tree = new DCDocComment(comment, firstSentence, body, tags);
+        final int savepos = pos;
-        tree.pos = pos;
+        Pair<List<DCTree>, List<DCTree>> pair = splitBody(fullBody);
+        DCDocComment tree = new DCDocComment(comment, fullBody, pair.fst, pair.snd, tags);
+        this.pos = tree.pos = savepos;
         return tree;
     }
 
@@ -273,4 +333,155 @@ public class DocTreeMaker {
         tree.pos = pos;
         return tree;
     }
+
+    public java.util.List<DocTree> getFirstSentence(java.util.List<? extends DocTree> list) {
+        Pair<List<DCTree>, List<DCTree>> pair = splitBody(list);
+        return new ArrayList<>(pair.fst);
+    }
+
+    /*
+     * Breaks up the body tags into the first sentence and its successors.
+     * The first sentence is determined with the presence of a period, block tag,
+     * or a sentence break, as returned by the BreakIterator. Trailing
+     * whitespaces are trimmed.
+     */
+    Pair<List<DCTree>, List<DCTree>> splitBody(Collection<? extends DocTree> list) {
+        ListBuffer<DCTree> body = new ListBuffer<>();
+        // split body into first sentence and body
+        ListBuffer<DCTree> fs = new ListBuffer<>();
+        if (list.isEmpty()) {
+            return new Pair<>(fs.toList(), body.toList());
+        }
+        boolean foundFirstSentence = false;
+        ArrayList<DocTree> alist = new ArrayList<>(list);
+        ListIterator<DocTree> itr = alist.listIterator();
+        while (itr.hasNext()) {
+            boolean isFirst = itr.previousIndex() == -1;
+            DocTree dt = itr.next();
+            int spos = ((DCTree)dt).pos;
+            if (foundFirstSentence) {
+                body.add((DCTree) dt);
+                continue;
+            }
+            switch (dt.getKind()) {
+                case TEXT:
+                    DCText tt = (DCText)dt;
+                    String s = tt.getBody();
+                    int sbreak = getSentenceBreak(s);
+                    if (sbreak > 0) {
+                        s = removeTrailingWhitespace(s.substring(0, sbreak));
+                        DCText text = this.at(spos).Text(s);
+                        fs.add(text);
+                        foundFirstSentence = true;
+                        int nwPos = skipWhiteSpace(tt.getBody(), sbreak);
+                        if (nwPos > 0) {
+                            DCText text2 = this.at(spos + nwPos).Text(tt.getBody().substring(nwPos));
+                            body.add(text2);
+                        }
+                        continue;
+                    } else if (itr.hasNext()) {
+                        // if the next doctree is a break, remove trailing spaces
+                        DocTree next = itr.next();
+                        boolean sbrk = isSentenceBreak(next, false);
+                        if (sbrk) {
+                            s = removeTrailingWhitespace(s);
+                            DCText text = this.at(spos).Text(s);
+                            fs.add(text);
+                            body.add((DCTree)next);
+                            foundFirstSentence = true;
+                            continue;
+                        }
+                        // reset to previous for further processing
+                        itr.previous();
+                    }
+                    break;
+                default:
+                    if (isSentenceBreak(dt, isFirst)) {
+                        body.add((DCTree)dt);
+                        foundFirstSentence = true;
+                        continue;
+                    }
+            }
+            fs.add((DCTree)dt);
+        }
+        return new Pair<>(fs.toList(), body.toList());
+    }
+
+    /*
+     * Computes the first sentence break.
+     */
+    int defaultSentenceBreak(String s) {
+        // scan for period followed by whitespace
+        int period = -1;
+        for (int i = 0; i < s.length(); i++) {
+            switch (s.charAt(i)) {
+                case '.':
+                    period = i;
+                    break;
+
+                case ' ':
+                case '\f':
+                case '\n':
+                case '\r':
+                case '\t':
+                    if (period >= 0) {
+                        return i;
+                    }
+                    break;
+
+                default:
+                    period = -1;
+                    break;
+            }
+        }
+        return -1;
+    }
+
+    int getSentenceBreak(String s) {
+        if (sentenceBreaker == null) {
+            return defaultSentenceBreak(s);
+        }
+        sentenceBreaker.setText(s);
+        return sentenceBreaker.first();
+    }
+
+    boolean isSentenceBreak(javax.lang.model.element.Name tagName) {
+        return sentenceBreakTags.contains(get(tagName));
+    }
+
+    boolean isSentenceBreak(DocTree dt, boolean isFirstDocTree) {
+        switch (dt.getKind()) {
+            case START_ELEMENT:
+                    StartElementTree set = (StartElementTree)dt;
+                    return !isFirstDocTree && ((DCTree) dt).pos > 1 && isSentenceBreak(set.getName());
+            case END_ELEMENT:
+                    EndElementTree eet = (EndElementTree)dt;
+                    return !isFirstDocTree && ((DCTree) dt).pos > 1 && isSentenceBreak(eet.getName());
+            default:
+                return false;
+        }
+    }
+
+    /*
+     * Returns the position of the the first non-white space
+     */
+    int skipWhiteSpace(String s, int start) {
+        for (int i = start; i < s.length(); i++) {
+            char c = s.charAt(i);
+            if (!Character.isWhitespace(c)) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    String removeTrailingWhitespace(String s) {
+        for (int i = s.length() - 1 ; i > 0 ; i--) {
+            char ch = s.charAt(i);
+            if (!Character.isWhitespace(ch)) {
+                return s.substring(0, i + 1);
+            }
+        }
+        return s;
+    }
 }

langtools/test/com/sun/javadoc/testSimpleTag/TestSimpleTag.java

@@ -51,7 +51,8 @@ public class TestSimpleTag extends JavadocTester {
                 "-tag", "regular:a:Regular Tag:",
                 "-tag", "back-slash\\:tag\\\\:a:Back-Slash-Tag:",
                 testSrc("C.java"));
-        checkExit(Exit.FAILED); // TODO: investigate why failed
+        // doclint fails because '\' is not allowed in tag name
+        checkExit(Exit.FAILED);
 
         checkOutput("C.html", true,
                 "<span class=\"simpleTagLabel\">Todo:</span>",

langtools/test/tools/javac/doctree/DocCommentTester.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -258,7 +258,6 @@ public class DocCommentTester {
                     } catch (IOException e) {
                         source = "";
                     }
-
                     // remove existing gold by removing all block comments after the first '{'.
                     int start = source.indexOf("{");
                     while ((start = source.indexOf("\n/*\n", start)) != -1) {
@@ -663,8 +662,6 @@ public class DocCommentTester {
                 else
                     return s;
             }
-
-
         }
     }
 
@@ -763,17 +760,15 @@ public class DocCommentTester {
          * Normalize white space in places where the tree does not preserve it.
          */
         String normalize(String s) {
-            return s.trim()
+            s = s.trim()
-                    .replaceFirst("\\.\\s++([^@])", ". $1")
                     .replaceFirst("\\.\\s*\\n *@", ".\n@")
-                    .replaceFirst("\\s+<(/?p|pre|h[1-6])>", " <$1>")
                     .replaceAll("\\{@docRoot\\s+\\}", "{@docRoot}")
                     .replaceAll("\\{@inheritDoc\\s+\\}", "{@inheritDoc}")
                     .replaceAll("(\\{@value\\s+[^}]+)\\s+(\\})", "$1$2")
-                    .replaceAll("\n[ \t]+@", "\n@");
+                    .replaceAll("\n[ \t]+@", "\n@")
+                    .replaceAll("(\\{@code)(\\x20)(\\s+.*)", "$1$3");
+            return s;
         }
-
     }
-
 }
 

langtools/test/tools/javac/doctree/ElementTest.java

@@ -23,7 +23,7 @@
 
 /*
  * @test
- * @bug 7021614
+ * @bug 7021614 8078320
  * @summary extend com.sun.source API to support parsing javadoc comments
  * @modules jdk.compiler/com.sun.tools.javac.api
  *          jdk.compiler/com.sun.tools.javac.file
@@ -40,13 +40,13 @@ class ElementTest {
     void simple() { }
 /*
 DocComment[DOC_COMMENT, pos:1
-  firstSentence: empty
+  firstSentence: 2
-  body: 3
     StartElement[START_ELEMENT, pos:1
       name:p
       attributes: empty
     ]
     Text[TEXT, pos:4, para]
+  body: 1
     EndElement[END_ELEMENT, pos:8, p]
   block tags: empty
 ]

langtools/test/tools/javac/doctree/FirstSentenceTest.java

@@ -23,7 +23,7 @@
 
 /*
  * @test
- * @bug 7021614
+ * @bug 7021614 8078320
  * @summary extend com.sun.source API to support parsing javadoc comments
  * @modules jdk.compiler/com.sun.tools.javac.api
  *          jdk.compiler/com.sun.tools.javac.file
@@ -114,6 +114,26 @@ DocComment[DOC_COMMENT, pos:1
     Text[TEXT, pos:17, jkl_mno_pqr]
   block tags: empty
 ]
+*/
+
+    /**
+     *
+     * <p>abc def ghi.
+     * jdl mno pqf
+     */
+    void newline_p() { }
+/*
+DocComment[DOC_COMMENT, pos:2
+  firstSentence: 2
+    StartElement[START_ELEMENT, pos:2
+      name:p
+      attributes: empty
+    ]
+    Text[TEXT, pos:5, abc_def_ghi.]
+  body: 1
+    Text[TEXT, pos:19, jdl_mno_pqf]
+  block tags: empty
+]
 */
 
     /**
@@ -197,6 +217,40 @@ DocComment[DOC_COMMENT, pos:1
     ]
 ]
 */
-
+    /**
+     * <p> abc def.
+     * ghi jkl
+     */
+    void p_at_zero() { }
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: 2
+    StartElement[START_ELEMENT, pos:1
+      name:p
+      attributes: empty
+    ]
+    Text[TEXT, pos:4, _abc_def.]
+  body: 1
+    Text[TEXT, pos:15, ghi_jkl]
+  block tags: empty
+]
+*/
+    /**
+     * abc <p> def. ghi jkl
+     */
+    void p_at_nonzero() { }
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: 1
+    Text[TEXT, pos:1, abc]
+  body: 2
+    StartElement[START_ELEMENT, pos:5
+      name:p
+      attributes: empty
+    ]
+    Text[TEXT, pos:8, _def._ghi_jkl]
+  block tags: empty
+]
+*/
 }
 

langtools/test/tools/javac/doctree/InPreTest.java

@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8078320
+ * @summary extend com.sun.source API to support parsing javadoc comments
+ * @modules jdk.compiler/com.sun.tools.javac.api
+ *          jdk.compiler/com.sun.tools.javac.file
+ *          jdk.compiler/com.sun.tools.javac.tree
+ *          jdk.compiler/com.sun.tools.javac.util
+ * @build DocCommentTester
+ * @run main DocCommentTester InPreTest.java
+ */
+
+class InPreTest {
+    /**
+     * xyz<pre> pqr </pre> abc{@code  def  }ghi
+     */
+    public void after_pre() { }
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: 1
+    Text[TEXT, pos:1, xyz]
+  body: 6
+    StartElement[START_ELEMENT, pos:4
+      name:pre
+      attributes: empty
+    ]
+    Text[TEXT, pos:9, _pqr_]
+    EndElement[END_ELEMENT, pos:14, pre]
+    Text[TEXT, pos:20, _abc]
+    Literal[CODE, pos:24, _def__]
+    Text[TEXT, pos:38, ghi]
+  block tags: empty
+]
+*/
+    /**
+     * abc{@code def}ghi
+     */
+    public void no_pre() { }
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: 3
+    Text[TEXT, pos:1, abc]
+    Literal[CODE, pos:4, def]
+    Text[TEXT, pos:15, ghi]
+  body: empty
+  block tags: empty
+]
+*/
+    /**
+     * xyz<pre> abc{@code  def  }ghi</pre>
+     */
+    public void pre_after_text() {}
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: 1
+    Text[TEXT, pos:1, xyz]
+  body: 5
+    StartElement[START_ELEMENT, pos:4
+      name:pre
+      attributes: empty
+    ]
+    Text[TEXT, pos:9, _abc]
+    Literal[CODE, pos:13, _def__]
+    Text[TEXT, pos:27, ghi]
+    EndElement[END_ELEMENT, pos:30, pre]
+  block tags: empty
+]
+*/
+
+    /**
+     * abc{@code  def  }ghi
+     */
+    public void no_pre_extra_whitespace() { }
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: 3
+    Text[TEXT, pos:1, abc]
+    Literal[CODE, pos:4, _def__]
+    Text[TEXT, pos:18, ghi]
+  body: empty
+  block tags: empty
+]
+*/
+    /**
+     * <pre> abc{@code  def  }ghi</pre>
+     */
+    public void in_pre() { }
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: 4
+    StartElement[START_ELEMENT, pos:1
+      name:pre
+      attributes: empty
+    ]
+    Text[TEXT, pos:6, _abc]
+    Literal[CODE, pos:10, _def__]
+    Text[TEXT, pos:24, ghi]
+  body: 1
+    EndElement[END_ELEMENT, pos:27, pre]
+  block tags: empty
+]
+*/
+    /**
+     * <pre> abc{@code
+     * def  }ghi</pre>
+     */
+    public void in_pre_with_space_nl() { }
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: 4
+    StartElement[START_ELEMENT, pos:1
+      name:pre
+      attributes: empty
+    ]
+    Text[TEXT, pos:6, _abc]
+    Literal[CODE, pos:10, |_def__]
+    Text[TEXT, pos:24, ghi]
+  body: 1
+    EndElement[END_ELEMENT, pos:27, pre]
+  block tags: empty
+]
+*/
+
+    /**
+     * <pre> abc{@code
+     *def  }ghi</pre>
+     */
+    public void in_pre_with_nl() { }
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: 4
+    StartElement[START_ELEMENT, pos:1
+      name:pre
+      attributes: empty
+    ]
+    Text[TEXT, pos:6, _abc]
+    Literal[CODE, pos:10, |def__]
+    Text[TEXT, pos:23, ghi]
+  body: 1
+    EndElement[END_ELEMENT, pos:26, pre]
+  block tags: empty
+]
+*/
+    /**
+     * abc {@code
+     */
+    public void bad_code_no_content() { }
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: 2
+    Text[TEXT, pos:1, abc_]
+    Erroneous[ERRONEOUS, pos:5
+      code: compiler.err.dc.unterminated.inline.tag
+      body: {@code
+    ]
+  body: empty
+  block tags: empty
+]
+*/
+    /**
+     * abc {@code abc
+     */
+    public void bad_code_content() { }
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: 2
+    Text[TEXT, pos:1, abc_]
+    Erroneous[ERRONEOUS, pos:5
+      code: compiler.err.dc.unterminated.inline.tag
+      body: {@code_abc
+    ]
+  body: empty
+  block tags: empty
+]
+*/
+}

langtools/test/tools/javac/doctree/TagTest.java

@@ -23,7 +23,7 @@
 
 /*
  * @test
- * @bug 7021614
+ * @bug 7021614 8078320
  * @summary extend com.sun.source API to support parsing javadoc comments
  * @modules jdk.compiler/com.sun.tools.javac.api
  *          jdk.compiler/com.sun.tools.javac.file
@@ -34,6 +34,40 @@
  */
 
 class TagTest {
+    /**
+     * @tag:colon abc
+     */
+    void custom_tag_with_a_colon() {}
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: empty
+  body: empty
+  block tags: 1
+    UnknownBlockTag[UNKNOWN_BLOCK_TAG, pos:1
+      tag:tag:colon
+      content: 1
+        Text[TEXT, pos:12, abc]
+    ]
+]
+*/
+
+    /**
+     * @tag-hyphen abc
+     */
+    void custom_tag_with_a_hyphen() {}
+/*
+DocComment[DOC_COMMENT, pos:1
+  firstSentence: empty
+  body: empty
+  block tags: 1
+    UnknownBlockTag[UNKNOWN_BLOCK_TAG, pos:1
+      tag:tag-hyphen
+      content: 1
+        Text[TEXT, pos:13, abc]
+    ]
+]
+*/
+
     /**
      * @author jjg
      */