stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

7104201: Refactor DocCommentScanner

Browse Source 
Add new Comment helper class to parse contents of comments in source code

Reviewed-by: jjg
This commit is contained in:
Maurizio Cimadamore 2011-11-04 12:36:40 +00:00
parent dae561e3ea
commit be5a83c8ce
7 changed files with 638 additions and 588 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

422
langtools/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java
View File

@ -25,10 +25,11 @@
package com.sun.tools.javac.parser;
import java.nio.CharBuffer;
import com.sun.tools.javac.code.Source;
import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
import com.sun.tools.javac.util.*;
import java.nio.CharBuffer;
import static com.sun.tools.javac.parser.Tokens.*;
import static com.sun.tools.javac.util.LayoutCharacters.*;
@ -65,9 +66,6 @@ public class JavaTokenizer {
*/
private final Log log;
/** The name table. */
private final Names names;
/** The token factory. */
private final Tokens tokens;
@ -87,18 +85,12 @@ public class JavaTokenizer {
*/
protected int errPos = Position.NOPOS;
/** Has a @deprecated been encountered in last doc comment?
* this needs to be reset by client.
/** The Unicode reader (low-level stream reader).
*/
protected boolean deprecatedFlag = false;
/** A character buffer for saved chars.
*/
protected char[] sbuf = new char[128];
protected int sp;
protected UnicodeReader reader;
protected ScannerFactory fac;
private static final boolean hexFloatsWork = hexFloatsWork();
private static boolean hexFloatsWork() {
try {
@ -129,14 +121,14 @@ public class JavaTokenizer {
}
protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {
log = fac.log;
names = fac.names;
tokens = fac.tokens;
source = fac.source;
this.fac = fac;
this.log = fac.log;
this.tokens = fac.tokens;
this.source = fac.source;
this.reader = reader;
allowBinaryLiterals = source.allowBinaryLiterals();
allowHexFloats = source.allowHexFloats();
allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
this.allowBinaryLiterals = source.allowBinaryLiterals();
this.allowHexFloats = source.allowHexFloats();
this.allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
}
/** Report an error at the given position using the provided arguments.
@ -147,38 +139,13 @@ public class JavaTokenizer {
errPos = pos;
}
/** Read next character in comment, skipping over double '\' characters.
*/
protected void scanCommentChar() {
reader.scanChar();
if (reader.ch == '\\') {
if (reader.peekChar() == '\\' && !reader.isUnicode()) {
reader.skipChar();
} else {
reader.convertUnicode();
}
}
}
/** Append a character to sbuf.
*/
private void putChar(char ch) {
if (sp == sbuf.length) {
char[] newsbuf = new char[sbuf.length * 2];
System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
sbuf = newsbuf;
}
sbuf[sp++] = ch;
}
/** Read next character in character or string literal and copy into sbuf.
*/
private void scanLitChar(int pos) {
if (reader.ch == '\\') {
if (reader.peekChar() == '\\' && !reader.isUnicode()) {
reader.skipChar();
putChar('\\');
reader.scanChar();
reader.putChar('\\', true);
} else {
reader.scanChar();
switch (reader.ch) {
@ -195,30 +162,30 @@ public class JavaTokenizer {
reader.scanChar();
}
}
putChar((char)oct);
reader.putChar((char)oct);
break;
case 'b':
putChar('\b'); reader.scanChar(); break;
reader.putChar('\b', true); break;
case 't':
putChar('\t'); reader.scanChar(); break;
reader.putChar('\t', true); break;
case 'n':
putChar('\n'); reader.scanChar(); break;
reader.putChar('\n', true); break;
case 'f':
putChar('\f'); reader.scanChar(); break;
reader.putChar('\f', true); break;
case 'r':
putChar('\r'); reader.scanChar(); break;
reader.putChar('\r', true); break;
case '\'':
putChar('\''); reader.scanChar(); break;
reader.putChar('\'', true); break;
case '\"':
putChar('\"'); reader.scanChar(); break;
reader.putChar('\"', true); break;
case '\\':
putChar('\\'); reader.scanChar(); break;
reader.putChar('\\', true); break;
default:
lexError(reader.bp, "illegal.esc.char");
}
}
} else if (reader.bp != reader.buflen) {
putChar(reader.ch); reader.scanChar();
reader.putChar(true);
}
}
@ -227,7 +194,7 @@ public class JavaTokenizer {
int savePos;
do {
if (reader.ch != '_') {
putChar(reader.ch);
reader.putChar(false);
} else {
if (!allowUnderscoresInLiterals) {
lexError(pos, "unsupported.underscore.lit", source.name);
@ -246,12 +213,10 @@ public class JavaTokenizer {
*/
private void scanHexExponentAndSuffix(int pos) {
if (reader.ch == 'p' || reader.ch == 'P') {
putChar(reader.ch);
reader.scanChar();
reader.putChar(true);
skipIllegalUnderscores();
if (reader.ch == '+' || reader.ch == '-') {
putChar(reader.ch);
reader.scanChar();
reader.putChar(true);
}
skipIllegalUnderscores();
if ('0' <= reader.ch && reader.ch <= '9') {
@ -268,14 +233,12 @@ public class JavaTokenizer {
lexError(pos, "malformed.fp.lit");
}
if (reader.ch == 'f' || reader.ch == 'F') {
putChar(reader.ch);
reader.scanChar();
reader.putChar(true);
tk = TokenKind.FLOATLITERAL;
radix = 16;
} else {
if (reader.ch == 'd' || reader.ch == 'D') {
putChar(reader.ch);
reader.scanChar();
reader.putChar(true);
}
tk = TokenKind.DOUBLELITERAL;
radix = 16;
@ -289,14 +252,12 @@ public class JavaTokenizer {
if ('0' <= reader.ch && reader.ch <= '9') {
scanDigits(pos, 10);
}
int sp1 = sp;
int sp1 = reader.sp;
if (reader.ch == 'e' || reader.ch == 'E') {
putChar(reader.ch);
reader.scanChar();
reader.putChar(true);
skipIllegalUnderscores();
if (reader.ch == '+' || reader.ch == '-') {
putChar(reader.ch);
reader.scanChar();
reader.putChar(true);
}
skipIllegalUnderscores();
if ('0' <= reader.ch && reader.ch <= '9') {
@ -304,7 +265,7 @@ public class JavaTokenizer {
return;
}
lexError(pos, "malformed.fp.lit");
sp = sp1;
reader.sp = sp1;
}
}
@ -314,13 +275,11 @@ public class JavaTokenizer {
radix = 10;
scanFraction(pos);
if (reader.ch == 'f' || reader.ch == 'F') {
putChar(reader.ch);
reader.scanChar();
reader.putChar(true);
tk = TokenKind.FLOATLITERAL;
} else {
if (reader.ch == 'd' || reader.ch == 'D') {
putChar(reader.ch);
reader.scanChar();
reader.putChar(true);
}
tk = TokenKind.DOUBLELITERAL;
}
@ -331,8 +290,7 @@ public class JavaTokenizer {
private void scanHexFractionAndSuffix(int pos, boolean seendigit) {
radix = 16;
Assert.check(reader.ch == '.');
putChar(reader.ch);
reader.scanChar();
reader.putChar(true);
skipIllegalUnderscores();
if (reader.digit(pos, 16) >= 0) {
seendigit = true;
@ -369,8 +327,7 @@ public class JavaTokenizer {
} else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) {
scanHexExponentAndSuffix(pos);
} else if (digitRadix == 10 && reader.ch == '.') {
putChar(reader.ch);
reader.scanChar();
reader.putChar(true);
scanFractionAndSuffix(pos);
} else if (digitRadix == 10 &&
(reader.ch == 'e' || reader.ch == 'E' ||
@ -393,10 +350,7 @@ public class JavaTokenizer {
boolean isJavaIdentifierPart;
char high;
do {
if (sp == sbuf.length) putChar(reader.ch); else sbuf[sp++] = reader.ch;
// optimization, was: putChar(reader.ch);
reader.scanChar();
reader.putChar(true);
switch (reader.ch) {
case 'A': case 'B': case 'C': case 'D': case 'E':
case 'F': case 'G': case 'H': case 'I': case 'J':
@ -423,7 +377,7 @@ public class JavaTokenizer {
break;
case '\u001A': // EOI is also a legal identifier part
if (reader.bp >= reader.buflen) {
name = names.fromChars(sbuf, 0, sp);
name = reader.name();
tk = tokens.lookupKind(name);
return;
}
@ -435,11 +389,7 @@ public class JavaTokenizer {
} else {
high = reader.scanSurrogates();
if (high != 0) {
if (sp == sbuf.length) {
putChar(high);
} else {
sbuf[sp++] = high;
}
reader.putChar(high);
isJavaIdentifierPart = Character.isJavaIdentifierPart(
Character.toCodePoint(high, reader.ch));
} else {
@ -447,7 +397,7 @@ public class JavaTokenizer {
}
}
if (!isJavaIdentifierPart) {
name = names.fromChars(sbuf, 0, sp);
name = reader.name();
tk = tokens.lookupKind(name);
return;
}
@ -474,11 +424,11 @@ public class JavaTokenizer {
*/
private void scanOperator() {
while (true) {
putChar(reader.ch);
Name newname = names.fromChars(sbuf, 0, sp);
reader.putChar(false);
Name newname = reader.name();
TokenKind tk1 = tokens.lookupKind(newname);
if (tk1 == TokenKind.IDENTIFIER) {
sp--;
reader.sp--;
break;
}
tk = tk1;
@ -487,111 +437,17 @@ public class JavaTokenizer {
}
}
/**
* Scan a documentation comment; determine if a deprecated tag is present.
* Called once the initial /, * have been skipped, positioned at the second *
* (which is treated as the beginning of the first line).
* Stops positioned at the closing '/'.
*/
@SuppressWarnings("fallthrough")
private void scanDocComment() {
boolean deprecatedPrefix = false;
forEachLine:
while (reader.bp < reader.buflen) {
// Skip optional WhiteSpace at beginning of line
while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
scanCommentChar();
}
// Skip optional consecutive Stars
while (reader.bp < reader.buflen && reader.ch == '*') {
scanCommentChar();
if (reader.ch == '/') {
return;
}
}
// Skip optional WhiteSpace after Stars
while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
scanCommentChar();
}
deprecatedPrefix = false;
// At beginning of line in the JavaDoc sense.
if (reader.bp < reader.buflen && reader.ch == '@' && !deprecatedFlag) {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'd') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'e') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'p') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'r') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'e') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'c') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'a') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 't') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'e') {
scanCommentChar();
if (reader.bp < reader.buflen && reader.ch == 'd') {
deprecatedPrefix = true;
scanCommentChar();
}}}}}}}}}}}
if (deprecatedPrefix && reader.bp < reader.buflen) {
if (Character.isWhitespace(reader.ch)) {
deprecatedFlag = true;
} else if (reader.ch == '*') {
scanCommentChar();
if (reader.ch == '/') {
deprecatedFlag = true;
return;
}
}
}
// Skip rest of line
while (reader.bp < reader.buflen) {
switch (reader.ch) {
case '*':
scanCommentChar();
if (reader.ch == '/') {
return;
}
break;
case CR: // (Spec 3.4)
scanCommentChar();
if (reader.ch != LF) {
continue forEachLine;
}
/* fall through to LF case */
case LF: // (Spec 3.4)
scanCommentChar();
continue forEachLine;
default:
scanCommentChar();
}
} // rest of line
} // forEachLine
return;
}
/** Read token.
*/
public Token readToken() {
sp = 0;
reader.sp = 0;
name = null;
deprecatedFlag = false;
radix = 0;
int pos = 0;
int endPos = 0;
List<Comment> comments = null;
try {
loop: while (true) {
@ -656,7 +512,7 @@ public class JavaTokenizer {
scanNumber(pos, 2);
}
} else {
putChar('0');
reader.putChar('0');
if (reader.ch == '_') {
int savePos = reader.bp;
do {
@ -676,14 +532,13 @@ public class JavaTokenizer {
case '.':
reader.scanChar();
if ('0' <= reader.ch && reader.ch <= '9') {
putChar('.');
reader.putChar('.');
scanFractionAndSuffix(pos);
} else if (reader.ch == '.') {
putChar('.'); putChar('.');
reader.scanChar();
reader.putChar('.'); reader.putChar('.', true);
if (reader.ch == '.') {
reader.scanChar();
putChar('.');
reader.putChar('.');
tk = TokenKind.ELLIPSIS;
} else {
lexError(pos, "malformed.fp.lit");
@ -712,32 +567,36 @@ public class JavaTokenizer {
reader.scanChar();
if (reader.ch == '/') {
do {
scanCommentChar();
reader.scanCommentChar();
} while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen);
if (reader.bp < reader.buflen) {
processComment(pos, reader.bp, CommentStyle.LINE);
comments = addDocReader(comments, processComment(pos, reader.bp, CommentStyle.LINE));
}
break;
} else if (reader.ch == '*') {
boolean isEmpty = false;
reader.scanChar();
CommentStyle style;
if (reader.ch == '*') {
style = CommentStyle.JAVADOC;
scanDocComment();
reader.scanCommentChar();
if (reader.ch == '/') {
isEmpty = true;
}
} else {
style = CommentStyle.BLOCK;
while (reader.bp < reader.buflen) {
if (reader.ch == '*') {
reader.scanChar();
if (reader.ch == '/') break;
} else {
scanCommentChar();
}
}
while (!isEmpty && reader.bp < reader.buflen) {
if (reader.ch == '*') {
reader.scanChar();
if (reader.ch == '/') break;
} else {
reader.scanCommentChar();
}
}
if (reader.ch == '/') {
reader.scanChar();
processComment(pos, reader.bp, style);
comments = addDocReader(comments, processComment(pos, reader.bp, style));
break;
} else {
lexError(pos, "unclosed.comment");
@ -789,11 +648,7 @@ public class JavaTokenizer {
} else {
char high = reader.scanSurrogates();
if (high != 0) {
if (sp == sbuf.length) {
putChar(high);
} else {
sbuf[sp++] = high;
}
reader.putChar(high);
isJavaIdentifierStart = Character.isJavaIdentifierStart(
Character.toCodePoint(high, reader.ch));
@ -816,10 +671,10 @@ public class JavaTokenizer {
}
endPos = reader.bp;
switch (tk.tag) {
case DEFAULT: return new Token(tk, pos, endPos, deprecatedFlag);
case NAMED: return new NamedToken(tk, pos, endPos, name, deprecatedFlag);
case STRING: return new StringToken(tk, pos, endPos, new String(sbuf, 0, sp), deprecatedFlag);
case NUMERIC: return new NumericToken(tk, pos, endPos, new String(sbuf, 0, sp), radix, deprecatedFlag);
case DEFAULT: return new Token(tk, pos, endPos, comments);
case NAMED: return new NamedToken(tk, pos, endPos, name, comments);
case STRING: return new StringToken(tk, pos, endPos, reader.chars(), comments);
case NUMERIC: return new NumericToken(tk, pos, endPos, reader.chars(), radix, comments);
default: throw new AssertionError();
}
}
@ -832,6 +687,12 @@ public class JavaTokenizer {
}
}
}
//where
List<Comment> addDocReader(List<Comment> docReaders, Comment docReader) {
return docReaders == null ?
List.of(docReader) :
docReaders.prepend(docReader);
}
/** Return the position where a lexical error occurred;
*/
@ -845,22 +706,18 @@ public class JavaTokenizer {
errPos = pos;
}
public enum CommentStyle {
LINE,
BLOCK,
JAVADOC,
}
/**
* Called when a complete comment has been scanned. pos and endPos
* will mark the comment boundary.
*/
protected void processComment(int pos, int endPos, CommentStyle style) {
protected Tokens.Comment processComment(int pos, int endPos, CommentStyle style) {
if (scannerDebug)
System.out.println("processComment(" + pos
+ "," + endPos + "," + style + ")=|"
+ new String(reader.getRawCharacters(pos, endPos))
+ "|");
char[] buf = reader.getRawCharacters(pos, endPos);
return new BasicComment<UnicodeReader>(new UnicodeReader(fac, buf, buf.length), style);
}
/**
@ -893,4 +750,125 @@ public class JavaTokenizer {
public Position.LineMap getLineMap() {
return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false);
}
/**
* Scan a documentation comment; determine if a deprecated tag is present.
* Called once the initial /, * have been skipped, positioned at the second *
* (which is treated as the beginning of the first line).
* Stops positioned at the closing '/'.
*/
protected class BasicComment<U extends UnicodeReader> implements Comment {
CommentStyle cs;
U comment_reader;
protected boolean deprecatedFlag = false;
protected boolean scanned = false;
protected BasicComment(U comment_reader, CommentStyle cs) {
this.comment_reader = comment_reader;
this.cs = cs;
}
public String getText() {
return null;
}
public CommentStyle getStyle() {
return cs;
}
public boolean isDeprecated() {
if (!scanned && cs == CommentStyle.JAVADOC) {
scanDocComment();
}
return deprecatedFlag;
}
@SuppressWarnings("fallthrough")
protected void scanDocComment() {
try {
boolean deprecatedPrefix = false;
comment_reader.bp += 3; // '/**'
comment_reader.ch = comment_reader.buf[comment_reader.bp];
forEachLine:
while (comment_reader.bp < comment_reader.buflen) {
// Skip optional WhiteSpace at beginning of line
while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
comment_reader.scanCommentChar();
}
// Skip optional consecutive Stars
while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '*') {
comment_reader.scanCommentChar();
if (comment_reader.ch == '/') {
return;
}
}
// Skip optional WhiteSpace after Stars
while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
comment_reader.scanCommentChar();
}
deprecatedPrefix = false;
// At beginning of line in the JavaDoc sense.
if (!deprecatedFlag) {
String deprecated = "@deprecated";
int i = 0;
while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == deprecated.charAt(i)) {
comment_reader.scanCommentChar();
i++;
if (i == deprecated.length()) {
deprecatedPrefix = true;
break;
}
}
}
if (deprecatedPrefix && comment_reader.bp < comment_reader.buflen) {
if (Character.isWhitespace(comment_reader.ch)) {
deprecatedFlag = true;
} else if (comment_reader.ch == '*') {
comment_reader.scanCommentChar();
if (comment_reader.ch == '/') {
deprecatedFlag = true;
return;
}
}
}
// Skip rest of line
while (comment_reader.bp < comment_reader.buflen) {
switch (comment_reader.ch) {
case '*':
comment_reader.scanCommentChar();
if (comment_reader.ch == '/') {
return;
}
break;
case CR: // (Spec 3.4)
comment_reader.scanCommentChar();
if (comment_reader.ch != LF) {
continue forEachLine;
}
/* fall through to LF case */
case LF: // (Spec 3.4)
comment_reader.scanCommentChar();
continue forEachLine;
default:
comment_reader.scanCommentChar();
}
} // rest of line
} // forEachLine
return;
} finally {
scanned = true;
}
}
}
}

23
langtools/src/share/classes/com/sun/tools/javac/parser/JavacParser.java
View File

@ -29,6 +29,7 @@ import java.util.*;
import com.sun.tools.javac.code.*;
import com.sun.tools.javac.parser.Tokens.*;
import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
import com.sun.tools.javac.tree.*;
import com.sun.tools.javac.tree.JCTree.*;
import com.sun.tools.javac.util.*;
@ -1584,7 +1585,7 @@ public class JavacParser implements Parser {
break;
case MONKEYS_AT:
case FINAL: {
String dc = token.docComment;
String dc = token.comment(CommentStyle.JAVADOC);
JCModifiers mods = modifiersOpt();
if (token.kind == INTERFACE ||
token.kind == CLASS ||
@ -1601,21 +1602,21 @@ public class JavacParser implements Parser {
break;
}
case ABSTRACT: case STRICTFP: {
String dc = token.docComment;
String dc = token.comment(CommentStyle.JAVADOC);
JCModifiers mods = modifiersOpt();
stats.append(classOrInterfaceOrEnumDeclaration(mods, dc));
break;
}
case INTERFACE:
case CLASS:
String dc = token.docComment;
String dc = token.comment(CommentStyle.JAVADOC);
stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(), dc));
break;
case ENUM:
case ASSERT:
if (allowEnums && token.kind == ENUM) {
error(token.pos, "local.enum");
dc = token.docComment;
dc = token.comment(CommentStyle.JAVADOC);
stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(), dc));
break;
} else if (allowAsserts && token.kind == ASSERT) {
@ -1991,7 +1992,7 @@ public class JavacParser implements Parser {
annotations.appendList(partial.annotations);
pos = partial.pos;
}
if (token.deprecatedFlag) {
if (token.deprecatedFlag()) {
flags |= Flags.DEPRECATED;
}
int lastPos = Position.NOPOS;
@ -2271,9 +2272,9 @@ public class JavacParser implements Parser {
seenImport = true;
defs.append(importDeclaration());
} else {
String docComment = token.docComment;
String docComment = token.comment(CommentStyle.JAVADOC);
if (firstTypeDecl && !seenImport && !seenPackage) {
docComment = firstToken.docComment;
docComment = firstToken.comment(CommentStyle.JAVADOC);
consumedToplevelDoc = true;
}
JCTree def = typeDeclaration(mods, docComment);
@ -2288,7 +2289,7 @@ public class JavacParser implements Parser {
}
JCTree.JCCompilationUnit toplevel = F.at(firstToken.pos).TopLevel(packageAnnotations, pid, defs.toList());
if (!consumedToplevelDoc)
attach(toplevel, firstToken.docComment);
attach(toplevel, firstToken.comment(CommentStyle.JAVADOC));
if (defs.elems.isEmpty())
storeEnd(toplevel, S.prevToken().endPos);
if (keepDocComments)
@ -2498,9 +2499,9 @@ public class JavacParser implements Parser {
/** EnumeratorDeclaration = AnnotationsOpt [TypeArguments] IDENTIFIER [ Arguments ] [ "{" ClassBody "}" ]
*/
JCTree enumeratorDeclaration(Name enumName) {
String dc = token.docComment;
String dc = token.comment(CommentStyle.JAVADOC);
int flags = Flags.PUBLIC|Flags.STATIC|Flags.FINAL|Flags.ENUM;
if (token.deprecatedFlag) {
if (token.deprecatedFlag()) {
flags |= Flags.DEPRECATED;
}
int pos = token.pos;
@ -2587,7 +2588,7 @@ public class JavacParser implements Parser {
nextToken();
return List.<JCTree>nil();
} else {
String dc = token.docComment;
String dc = token.comment(CommentStyle.JAVADOC);
int pos = token.pos;
JCModifiers mods = modifiersOpt();
if (token.kind == CLASS ||

617
langtools/src/share/classes/com/sun/tools/javac/parser/JavadocTokenizer.java
View File

@ -25,8 +25,8 @@
package com.sun.tools.javac.parser;
import com.sun.tools.javac.file.JavacFileManager;
import com.sun.tools.javac.parser.Tokens.Token;
import com.sun.tools.javac.parser.Tokens.Comment;
import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
import com.sun.tools.javac.util.*;
import java.nio.*;
@ -59,352 +59,295 @@ public class JavadocTokenizer extends JavaTokenizer {
super(fac, input, inputLength);
}
/** The comment input buffer, index of next chacter to be read,
* index of one past last character in buffer.
*/
private char[] buf;
private int bp;
private int buflen;
/** The current character.
*/
private char ch;
/** The column number position of the current character.
*/
private int col;
/** The buffer index of the last converted Unicode character
*/
private int unicodeConversionBp = 0;
/**
* Buffer for doc comment.
*/
private char[] docCommentBuffer = new char[1024];
/**
* Number of characters in doc comment buffer.
*/
private int docCommentCount;
/**
* Translated and stripped contents of doc comment
*/
private String docComment = null;
/** Unconditionally expand the comment buffer.
*/
private void expandCommentBuffer() {
char[] newBuffer = new char[docCommentBuffer.length * 2];
System.arraycopy(docCommentBuffer, 0, newBuffer,
0, docCommentBuffer.length);
docCommentBuffer = newBuffer;
@Override
protected Comment processComment(int pos, int endPos, CommentStyle style) {
char[] buf = reader.getRawCharacters(pos, endPos);
return new JavadocComment(new ColReader(fac, buf, buf.length), style);
}
/** Convert an ASCII digit from its base (8, 10, or 16)
* to its value.
/**
* This is a specialized version of UnicodeReader that keeps track of the
* column position within a given character stream (used for Javadoc processing).
*/
private int digit(int base) {
char c = ch;
int result = Character.digit(c, base);
if (result >= 0 && c > 0x7f) {
ch = "0123456789abcdef".charAt(result);
static class ColReader extends UnicodeReader {
int col;
ColReader(ScannerFactory fac, char[] input, int inputLength) {
super(fac, input, inputLength);
}
@Override
protected void convertUnicode() {
if (ch == '\\' && unicodeConversionBp != bp) {
bp++; ch = buf[bp]; col++;
if (ch == 'u') {
do {
bp++; ch = buf[bp]; col++;
} while (ch == 'u');
int limit = bp + 3;
if (limit < buflen) {
int d = digit(bp, 16);
int code = d;
while (bp < limit && d >= 0) {
bp++; ch = buf[bp]; col++;
d = digit(bp, 16);
code = (code << 4) + d;
}
if (d >= 0) {
ch = (char)code;
unicodeConversionBp = bp;
return;
}
}
// "illegal.Unicode.esc", reported by base scanner
} else {
bp--;
ch = '\\';
col--;
}
}
}
@Override
protected void scanCommentChar() {
scanChar();
if (ch == '\\') {
if (peekChar() == '\\' && !isUnicode()) {
putChar(ch, false);
bp++; col++;
} else {
convertUnicode();
}
}
}
@Override
protected void scanChar() {
bp++;
ch = buf[bp];
switch (ch) {
case '\r': // return
col = 0;
break;
case '\n': // newline
if (bp == 0 || buf[bp-1] != '\r') {
col = 0;
}
break;
case '\t': // tab
col = (col / TabInc * TabInc) + TabInc;
break;
case '\\': // possible Unicode
col++;
convertUnicode();
break;
default:
col++;
break;
}
}
}
protected class JavadocComment extends JavaTokenizer.BasicComment<ColReader> {
/**
* Translated and stripped contents of doc comment
*/
private String docComment = null;
JavadocComment(ColReader comment_reader, CommentStyle cs) {
super(comment_reader, cs);
}
return result;
}
/** Convert Unicode escape; bp points to initial '\' character
* (Spec 3.3).
*/
private void convertUnicode() {
if (ch == '\\' && unicodeConversionBp != bp) {
bp++; ch = buf[bp]; col++;
if (ch == 'u') {
do {
bp++; ch = buf[bp]; col++;
} while (ch == 'u');
int limit = bp + 3;
if (limit < buflen) {
int d = digit(16);
int code = d;
while (bp < limit && d >= 0) {
bp++; ch = buf[bp]; col++;
d = digit(16);
code = (code << 4) + d;
}
if (d >= 0) {
ch = (char)code;
unicodeConversionBp = bp;
return;
}
public String getText() {
if (!scanned && cs == CommentStyle.JAVADOC) {
scanDocComment();
}
return docComment;
}
@Override
@SuppressWarnings("fallthrough")
protected void scanDocComment() {
try {
boolean firstLine = true;
// Skip over first slash
comment_reader.scanCommentChar();
// Skip over first star
comment_reader.scanCommentChar();
// consume any number of stars
while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '*') {
comment_reader.scanCommentChar();
}
// is the comment in the form /**/, /***/, /****/, etc. ?
if (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '/') {
docComment = "";
return;
}
// skip a newline on the first line of the comment.
if (comment_reader.bp < comment_reader.buflen) {
if (comment_reader.ch == LF) {
comment_reader.scanCommentChar();
firstLine = false;
} else if (comment_reader.ch == CR) {
comment_reader.scanCommentChar();
if (comment_reader.ch == LF) {
comment_reader.scanCommentChar();
firstLine = false;
}
}
}
outerLoop:
// The outerLoop processes the doc comment, looping once
// for each line. For each line, it first strips off
// whitespace, then it consumes any stars, then it
// puts the rest of the line into our buffer.
while (comment_reader.bp < comment_reader.buflen) {
// The wsLoop consumes whitespace from the beginning
// of each line.
wsLoop:
while (comment_reader.bp < comment_reader.buflen) {
switch(comment_reader.ch) {
case ' ':
comment_reader.scanCommentChar();
break;
case '\t':
comment_reader.col = ((comment_reader.col - 1) / TabInc * TabInc) + TabInc;
comment_reader.scanCommentChar();
break;
case FF:
comment_reader.col = 0;
comment_reader.scanCommentChar();
break;
// Treat newline at beginning of line (blank line, no star)
// as comment text. Old Javadoc compatibility requires this.
/*---------------------------------*
case CR: // (Spec 3.4)
doc_reader.scanCommentChar();
if (ch == LF) {
col = 0;
doc_reader.scanCommentChar();
}
break;
case LF: // (Spec 3.4)
doc_reader.scanCommentChar();
break;
*---------------------------------*/
default:
// we've seen something that isn't whitespace;
// jump out.
break wsLoop;
}
}
// Are there stars here? If so, consume them all
// and check for the end of comment.
if (comment_reader.ch == '*') {
// skip all of the stars
do {
comment_reader.scanCommentChar();
} while (comment_reader.ch == '*');
// check for the closing slash.
if (comment_reader.ch == '/') {
// We're done with the doc comment
// scanChar() and breakout.
break outerLoop;
}
} else if (! firstLine) {
//The current line does not begin with a '*' so we will indent it.
for (int i = 1; i < comment_reader.col; i++) {
comment_reader.putChar(' ', false);
}
}
// The textLoop processes the rest of the characters
// on the line, adding them to our buffer.
textLoop:
while (comment_reader.bp < comment_reader.buflen) {
switch (comment_reader.ch) {
case '*':
// Is this just a star? Or is this the
// end of a comment?
comment_reader.scanCommentChar();
if (comment_reader.ch == '/') {
// This is the end of the comment,
// set ch and return our buffer.
break outerLoop;
}
// This is just an ordinary star. Add it to
// the buffer.
comment_reader.putChar('*', false);
break;
case ' ':
case '\t':
comment_reader.putChar(comment_reader.ch, false);
comment_reader.scanCommentChar();
break;
case FF:
comment_reader.scanCommentChar();
break textLoop; // treat as end of line
case CR: // (Spec 3.4)
comment_reader.scanCommentChar();
if (comment_reader.ch != LF) {
// Canonicalize CR-only line terminator to LF
comment_reader.putChar((char)LF, false);
break textLoop;
}
/* fall through to LF case */
case LF: // (Spec 3.4)
// We've seen a newline. Add it to our
// buffer and break out of this loop,
// starting fresh on a new line.
comment_reader.putChar(comment_reader.ch, false);
comment_reader.scanCommentChar();
break textLoop;
default:
// Add the character to our buffer.
comment_reader.putChar(comment_reader.ch, false);
comment_reader.scanCommentChar();
}
} // end textLoop
firstLine = false;
} // end outerLoop
if (comment_reader.sp > 0) {
int i = comment_reader.sp - 1;
trailLoop:
while (i > -1) {
switch (comment_reader.sbuf[i]) {
case '*':
i--;
break;
default:
break trailLoop;
}
}
comment_reader.sp = i + 1;
// Store the text of the doc comment
docComment = comment_reader.chars();
} else {
docComment = "";
}
} finally {
scanned = true;
if (docComment != null &&
docComment.matches("(?sm).*^\\s*@deprecated( |$).*")) {
deprecatedFlag = true;
}
// "illegal.Unicode.esc", reported by base scanner
} else {
bp--;
ch = '\\';
col--;
}
}
}
/** Read next character.
*/
private void scanChar() {
bp++;
ch = buf[bp];
switch (ch) {
case '\r': // return
col = 0;
break;
case '\n': // newline
if (bp == 0 || buf[bp-1] != '\r') {
col = 0;
}
break;
case '\t': // tab
col = (col / TabInc * TabInc) + TabInc;
break;
case '\\': // possible Unicode
col++;
convertUnicode();
break;
default:
col++;
break;
}
}
@Override
public Token readToken() {
docComment = null;
Token tk = super.readToken();
tk.docComment = docComment;
return tk;
}
/**
* Read next character in doc comment, skipping over double '\' characters.
* If a double '\' is skipped, put in the buffer and update buffer count.
*/
private void scanDocCommentChar() {
scanChar();
if (ch == '\\') {
if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
if (docCommentCount == docCommentBuffer.length)
expandCommentBuffer();
docCommentBuffer[docCommentCount++] = ch;
bp++; col++;
} else {
convertUnicode();
}
}
}
/**
* Process a doc comment and make the string content available.
* Strips leading whitespace and stars.
*/
@SuppressWarnings("fallthrough")
protected void processComment(int pos, int endPos, CommentStyle style) {
if (style != CommentStyle.JAVADOC) {
return;
}
buf = reader.getRawCharacters(pos, endPos);
buflen = buf.length;
bp = 0;
col = 0;
docCommentCount = 0;
boolean firstLine = true;
// Skip over first slash
scanDocCommentChar();
// Skip over first star
scanDocCommentChar();
// consume any number of stars
while (bp < buflen && ch == '*') {
scanDocCommentChar();
}
// is the comment in the form /**/, /***/, /****/, etc. ?
if (bp < buflen && ch == '/') {
docComment = "";
return;
}
// skip a newline on the first line of the comment.
if (bp < buflen) {
if (ch == LF) {
scanDocCommentChar();
firstLine = false;
} else if (ch == CR) {
scanDocCommentChar();
if (ch == LF) {
scanDocCommentChar();
firstLine = false;
}
}
}
outerLoop:
// The outerLoop processes the doc comment, looping once
// for each line. For each line, it first strips off
// whitespace, then it consumes any stars, then it
// puts the rest of the line into our buffer.
while (bp < buflen) {
// The wsLoop consumes whitespace from the beginning
// of each line.
wsLoop:
while (bp < buflen) {
switch(ch) {
case ' ':
scanDocCommentChar();
break;
case '\t':
col = ((col - 1) / TabInc * TabInc) + TabInc;
scanDocCommentChar();
break;
case FF:
col = 0;
scanDocCommentChar();
break;
// Treat newline at beginning of line (blank line, no star)
// as comment text. Old Javadoc compatibility requires this.
/*---------------------------------*
case CR: // (Spec 3.4)
scanDocCommentChar();
if (ch == LF) {
col = 0;
scanDocCommentChar();
}
break;
case LF: // (Spec 3.4)
scanDocCommentChar();
break;
*---------------------------------*/
default:
// we've seen something that isn't whitespace;
// jump out.
break wsLoop;
}
}
// Are there stars here? If so, consume them all
// and check for the end of comment.
if (ch == '*') {
// skip all of the stars
do {
scanDocCommentChar();
} while (ch == '*');
// check for the closing slash.
if (ch == '/') {
// We're done with the doc comment
// scanChar() and breakout.
break outerLoop;
}
} else if (! firstLine) {
//The current line does not begin with a '*' so we will indent it.
for (int i = 1; i < col; i++) {
if (docCommentCount == docCommentBuffer.length)
expandCommentBuffer();
docCommentBuffer[docCommentCount++] = ' ';
}
}
// The textLoop processes the rest of the characters
// on the line, adding them to our buffer.
textLoop:
while (bp < buflen) {
switch (ch) {
case '*':
// Is this just a star? Or is this the
// end of a comment?
scanDocCommentChar();
if (ch == '/') {
// This is the end of the comment,
// set ch and return our buffer.
break outerLoop;
}
// This is just an ordinary star. Add it to
// the buffer.
if (docCommentCount == docCommentBuffer.length)
expandCommentBuffer();
docCommentBuffer[docCommentCount++] = '*';
break;
case ' ':
case '\t':
if (docCommentCount == docCommentBuffer.length)
expandCommentBuffer();
docCommentBuffer[docCommentCount++] = ch;
scanDocCommentChar();
break;
case FF:
scanDocCommentChar();
break textLoop; // treat as end of line
case CR: // (Spec 3.4)
scanDocCommentChar();
if (ch != LF) {
// Canonicalize CR-only line terminator to LF
if (docCommentCount == docCommentBuffer.length)
expandCommentBuffer();
docCommentBuffer[docCommentCount++] = (char)LF;
break textLoop;
}
/* fall through to LF case */
case LF: // (Spec 3.4)
// We've seen a newline. Add it to our
// buffer and break out of this loop,
// starting fresh on a new line.
if (docCommentCount == docCommentBuffer.length)
expandCommentBuffer();
docCommentBuffer[docCommentCount++] = ch;
scanDocCommentChar();
break textLoop;
default:
// Add the character to our buffer.
if (docCommentCount == docCommentBuffer.length)
expandCommentBuffer();
docCommentBuffer[docCommentCount++] = ch;
scanDocCommentChar();
}
} // end textLoop
firstLine = false;
} // end outerLoop
if (docCommentCount > 0) {
int i = docCommentCount - 1;
trailLoop:
while (i > -1) {
switch (docCommentBuffer[i]) {
case '*':
i--;
break;
default:
break trailLoop;
}
}
docCommentCount = i + 1;
// Store the text of the doc comment
docComment = new String(docCommentBuffer, 0 , docCommentCount);
} else {
docComment = "";
}
}
/** Build a map for translating between line numbers and
* positions in the input.
*
* @return a LineMap */
public Position.LineMap getLineMap() {
char[] buf = reader.getRawCharacters();
return Position.makeLineMap(buf, buf.length, true);

83
langtools/src/share/classes/com/sun/tools/javac/parser/Tokens.java
View File

@ -30,8 +30,10 @@ import java.util.Locale;
import com.sun.tools.javac.api.Formattable;
import com.sun.tools.javac.api.Messages;
import com.sun.tools.javac.parser.Tokens.Token.Tag;
import com.sun.tools.javac.util.List;
import com.sun.tools.javac.util.Name;
import com.sun.tools.javac.util.Context;
import com.sun.tools.javac.util.ListBuffer;
import com.sun.tools.javac.util.Names;
/** A class that defines codes/utilities for Java source tokens
@ -281,6 +283,19 @@ public class Tokens {
}
}
public interface Comment {
enum CommentStyle {
LINE,
BLOCK,
JAVADOC,
}
String getText();
CommentStyle getStyle();
boolean isDeprecated();
}
/**
* This is the class representing a javac token. Each token has several fields
* that are set by the javac lexer (i.e. start/end position, string value, etc).
@ -304,18 +319,14 @@ public class Tokens {
/** The end position of this token */
public final int endPos;
/** Is this token preceeded by a deprecated comment? */
public final boolean deprecatedFlag;
/** Comment reader associated with this token */
public final List<Comment> comments;
/** Is this token preceeded by a deprecated comment? */
public String docComment;
Token(TokenKind kind, int pos, int endPos,
boolean deprecatedFlag) {
Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {
this.kind = kind;
this.pos = pos;
this.endPos = endPos;
this.deprecatedFlag = deprecatedFlag;
this.comments = comments;
checkKind();
}
@ -331,8 +342,8 @@ public class Tokens {
throw new AssertionError("Cant split - bad subtokens");
}
return new Token[] {
new Token(t1, pos, pos + t1.name.length(), deprecatedFlag),
new Token(t2, pos + t1.name.length(), endPos, false)
new Token(t1, pos, pos + t1.name.length(), comments),
new Token(t2, pos + t1.name.length(), endPos, null)
};
}
@ -353,14 +364,52 @@ public class Tokens {
public int radix() {
throw new UnsupportedOperationException();
}
/**
* Preserve classic semantics - if multiple javadocs are found on the token
* the last one is returned
*/
public String comment(Comment.CommentStyle style) {
List<Comment> readers = getReaders(Comment.CommentStyle.JAVADOC);
return readers.isEmpty() ?
null :
readers.head.getText();
}
/**
* Preserve classic semantics - deprecated should be set if at least one
* javadoc comment attached to this token contains the '@deprecated' string
*/
public boolean deprecatedFlag() {
for (Comment r : getReaders(Comment.CommentStyle.JAVADOC)) {
if (r.isDeprecated()) {
return true;
}
}
return false;
}
private List<Comment> getReaders(Comment.CommentStyle style) {
if (comments == null) {
return List.nil();
} else {
ListBuffer<Comment> buf = ListBuffer.lb();
for (Comment r : comments) {
if (r.getStyle() == style) {
buf.add(r);
}
}
return buf.toList();
}
}
}
final static class NamedToken extends Token {
/** The name of this token */
public final Name name;
public NamedToken(TokenKind kind, int pos, int endPos, Name name, boolean deprecatedFlag) {
super(kind, pos, endPos, deprecatedFlag);
public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {
super(kind, pos, endPos, comments);
this.name = name;
}
@ -380,8 +429,8 @@ public class Tokens {
/** The string value of this token */
public final String stringVal;
public StringToken(TokenKind kind, int pos, int endPos, String stringVal, boolean deprecatedFlag) {
super(kind, pos, endPos, deprecatedFlag);
public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {
super(kind, pos, endPos, comments);
this.stringVal = stringVal;
}
@ -401,8 +450,8 @@ public class Tokens {
/** The 'radix' value of this token */
public final int radix;
public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, boolean deprecatedFlag) {
super(kind, pos, endPos, stringVal, deprecatedFlag);
public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {
super(kind, pos, endPos, stringVal, comments);
this.radix = radix;
}
@ -419,5 +468,5 @@ public class Tokens {
}
public static final Token DUMMY =
new Token(TokenKind.ERROR, 0, 0, false);
new Token(TokenKind.ERROR, 0, 0, null);
}

55
langtools/src/share/classes/com/sun/tools/javac/parser/UnicodeReader.java
View File

@ -26,8 +26,12 @@
package com.sun.tools.javac.parser;
import com.sun.tools.javac.file.JavacFileManager;
import java.nio.CharBuffer;
import com.sun.tools.javac.util.Log;
import com.sun.tools.javac.util.Name;
import com.sun.tools.javac.util.Names;
import java.nio.CharBuffer;
import static com.sun.tools.javac.util.LayoutCharacters.*;
/** The char reader used by the javac lexer/tokenizer. Returns the sequence of
@ -58,6 +62,12 @@ public class UnicodeReader {
protected int unicodeConversionBp = -1;
protected Log log;
protected Names names;
/** A character buffer for saved chars.
*/
protected char[] sbuf = new char[128];
protected int sp;
/**
* Create a scanner from the input array. This method might
@ -76,6 +86,7 @@ public class UnicodeReader {
protected UnicodeReader(ScannerFactory sf, char[] input, int inputLength) {
log = sf.log;
names = sf.names;
if (inputLength == input.length) {
if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) {
inputLength--;
@ -103,6 +114,48 @@ public class UnicodeReader {
}
}
/** Read next character in comment, skipping over double '\' characters.
*/
protected void scanCommentChar() {
scanChar();
if (ch == '\\') {
if (peekChar() == '\\' && !isUnicode()) {
skipChar();
} else {
convertUnicode();
}
}
}
/** Append a character to sbuf.
*/
protected void putChar(char ch, boolean scan) {
if (sp == sbuf.length) {
char[] newsbuf = new char[sbuf.length * 2];
System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
sbuf = newsbuf;
}
sbuf[sp++] = ch;
if (scan)
scanChar();
}
protected void putChar(char ch) {
putChar(ch, false);
}
protected void putChar(boolean scan) {
putChar(ch, scan);
}
Name name() {
return names.fromChars(sbuf, 0, sp);
}
String chars() {
return new String(sbuf, 0, sp);
}
/** Convert unicode escape; bp points to initial '\' character
* (Spec 3.3).
*/

20
langtools/test/tools/javac/depDocComment/DeprecatedDocComment4.java Normal file
View File

@ -0,0 +1,20 @@
/**
* @test /nodynamiccopyright/
* @bug 7104201
* @summary Refactor DocCommentScanner
* @compile/fail/ref=DeprecatedDocComment4.out -XDrawDiagnostics -Werror -Xlint:dep-ann DeprecatedDocComment4.java
*/
class DeprecatedDocComment4 {
/** @deprecated **/
/* block */
void test1() {};
/** @deprecated **/
/** double javadoc */
void test2() {};
/** @deprecated **/
//line comment
void test3() {};
}

6
langtools/test/tools/javac/depDocComment/DeprecatedDocComment4.out Normal file
View File

@ -0,0 +1,6 @@
DeprecatedDocComment4.java:11:10: compiler.warn.missing.deprecated.annotation
DeprecatedDocComment4.java:15:10: compiler.warn.missing.deprecated.annotation
DeprecatedDocComment4.java:19:10: compiler.warn.missing.deprecated.annotation
- compiler.err.warnings.and.werror
1 error
3 warnings