azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

KDoc lexer refactored (got rid of hacks and separate tokens for words). Fixed bug with isInComment(element) detection.

Browse Source
This commit is contained in:
Sergey Rostov
2013-05-25 15:22:52 +04:00
committed by Natalia.Ukhorskaya
parent 85ab1619a7
commit 940008cdaa
14 changed files with 97 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/frontend/src/org/jetbrains/jet/kdoc/lexer/KDoc.flex
+16 -25
View File
@@ -23,18 +23,6 @@ import com.intellij.util.text.CharArrayUtil;
private final Boolean yytextContainLineBreaks() {
return CharArrayUtil.containLineBreaks(zzBuffer, zzStartRead, zzMarkedPos);
}
private final void pushbackEnd() {
if (isLastToken() && CharArrayUtil.regionMatches(zzBuffer, zzMarkedPos - 2, zzMarkedPos, "*/")) {
yypushback(2);
}
}
private final void pushbackText() {
int i = zzStartRead;
while (zzBuffer.charAt(i) == '*') i++;
yypushback(zzMarkedPos - i);
}
%}
%function advance
@@ -54,22 +42,25 @@ NOT_WHITE_SPACE_CHAR=[^\ \t\f\n\r]
<YYINITIAL> "/**" { yybegin(CONTENTS);
return KDocTokens.START; }
"*"+ "/" { if (isLastToken())
return KDocTokens.END; }
"*"+ "/" { if (isLastToken()) return KDocTokens.END;
else return KDocTokens.TEXT; }
// hack: make longest match
<LINE_BEGINNING> "*"+ {NOT_WHITE_SPACE_CHAR}* { pushbackText();
yybegin(CONTENTS);
return KDocTokens.LEADING_ASTERISK; }
<LINE_BEGINNING> "*"+ { yybegin(CONTENTS);
return KDocTokens.LEADING_ASTERISK; }
<CONTENTS, LINE_BEGINNING> {
// todo: put markdown and @tags token patterns here
{WHITE_SPACE_CHAR}+ {
if (yytextContainLineBreaks()) {
yybegin(LINE_BEGINNING);
return TokenType.WHITE_SPACE;
} else {
yybegin(CONTENTS);
return KDocTokens.TEXT; // internal white space
}
}
{NOT_WHITE_SPACE_CHAR}+ { pushbackEnd();
return KDocTokens.TEXT; }
{WHITE_SPACE_CHAR}+ { if (yytextContainLineBreaks())
yybegin(LINE_BEGINNING);
return TokenType.WHITE_SPACE; }
. { yybegin(CONTENTS);
return KDocTokens.TEXT; }
}
. { return TokenType.BAD_CHARACTER; }
. { return TokenType.BAD_CHARACTER; }
compiler/frontend/src/org/jetbrains/jet/kdoc/lexer/KDocLexer.java
+9 -2
View File
@@ -17,11 +17,18 @@
package org.jetbrains.jet.kdoc.lexer;
import com.intellij.lexer.FlexAdapter;
import com.intellij.lexer.MergingLexerAdapter;
import com.intellij.psi.tree.TokenSet;
import java.io.Reader;
public class KDocLexer extends FlexAdapter {
public class KDocLexer extends MergingLexerAdapter {
public KDocLexer() {
super(new _KDocLexer((Reader) null));
super(
new FlexAdapter(
new _KDocLexer((Reader) null)
),
TokenSet.create(KDocTokens.TEXT)
);
}
}
compiler/frontend/src/org/jetbrains/jet/kdoc/lexer/KDocTokens.java
+1 -4
View File
@@ -23,7 +23,6 @@ import com.intellij.lang.PsiParser;
import com.intellij.openapi.project.Project;
import com.intellij.psi.PsiElement;
import com.intellij.psi.tree.ILazyParseableElementType;
import com.intellij.psi.tree.TokenSet;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jet.kdoc.parser.KDocParser;
import org.jetbrains.jet.kdoc.psi.impl.KDocImpl;
@@ -53,7 +52,5 @@ public interface KDocTokens {
KDocToken END = new KDocToken("KDOC_END");
KDocToken LEADING_ASTERISK = new KDocToken("KDOC_LEADING_ASTERISK");
KDocToken TEXT = new KDocToken("KDOC_TEXT");
TokenSet ALL_KDOC_TOKENS = TokenSet.create(START, END, LEADING_ASTERISK, TEXT);
KDocToken TEXT = new KDocToken("KDOC_TEXT");
}
compiler/frontend/src/org/jetbrains/jet/kdoc/lexer/_KDocLexer.java
+30 -39
View File
@@ -1,4 +1,4 @@
/* The following code was generated by JFlex 1.4.3 on 23.05.13 19:26 */
/* The following code was generated by JFlex 1.4.3 on 25.05.13 15:08 */
package org.jetbrains.jet.kdoc.lexer;
@@ -11,7 +11,7 @@ import com.intellij.util.text.CharArrayUtil;
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.3
* on 23.05.13 19:26 from the specification file
* on 25.05.13 15:08 from the specification file
* <tt>/Users/factitious/Documents/kotlin/compiler/frontend/src/org/jetbrains/jet/kdoc/lexer/KDoc.flex</tt>
*/
class _KDocLexer implements FlexLexer {
@@ -19,9 +19,11 @@ class _KDocLexer implements FlexLexer {
private static final int ZZ_BUFFERSIZE = 16384;
/** lexical states */
public static final int LINE_BEGINNING = 4;
public static final int LINE_BEGINNING = 8;
public static final int STRONG_CONTENTS = 6;
public static final int CONTENTS = 2;
public static final int YYINITIAL = 0;
public static final int EMPHASIS_CONTENTS = 4;
/**
* ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l
@@ -30,7 +32,7 @@ class _KDocLexer implements FlexLexer {
* l is of the form l = 2*k, k a non negative integer
*/
private static final int ZZ_LEXSTATE[] = {
0, 0, 1, 1, 2, 2
0, 0, 1, 1, 1, 1, 1, 1, 2, 2
};
/**
@@ -52,10 +54,10 @@ class _KDocLexer implements FlexLexer {
private static final String ZZ_ACTION_PACKED_0 =
"\3\0\3\1\1\2\1\3\1\2\1\4\1\0\1\5"+
"\1\0\1\5\1\4\1\5\1\6";
"\1\0\1\6";
private static int [] zzUnpackAction() {
int [] result = new int[17];
int [] result = new int[14];
int offset = 0;
offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
return result;
@@ -80,12 +82,11 @@ class _KDocLexer implements FlexLexer {
private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
private static final String ZZ_ROWMAP_PACKED_0 =
"\0\0\0\5\0\12\0\17\0\24\0\31\0\36\0\43"+
"\0\50\0\55\0\62\0\17\0\31\0\36\0\67\0\67"+
"\0\17";
"\0\0\0\5\0\12\0\17\0\24\0\31\0\17\0\36"+
"\0\31\0\43\0\50\0\17\0\31\0\17";
private static int [] zzUnpackRowMap() {
int [] result = new int[17];
int [] result = new int[14];
int offset = 0;
offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
return result;
@@ -110,13 +111,11 @@ class _KDocLexer implements FlexLexer {
private static final String ZZ_TRANS_PACKED_0 =
"\2\4\1\5\1\6\1\0\1\7\1\10\1\7\1\11"+
"\1\10\1\7\1\10\1\7\1\12\1\10\10\0\1\13"+
"\3\0\1\14\1\15\1\0\1\7\1\0\2\7\2\0"+
"\1\10\2\0\1\10\1\7\1\0\1\16\1\11\1\0"+
"\1\17\1\0\1\20\1\12\4\0\1\21\1\0\1\17"+
"\1\0\2\17\1\0";
"\3\0\1\14\1\15\2\0\1\10\2\0\1\10\2\0"+
"\1\14\1\12\4\0\1\16\1\0";
private static int [] zzUnpackTrans() {
int [] result = new int[60];
int [] result = new int[45];
int offset = 0;
offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
return result;
@@ -157,10 +156,11 @@ class _KDocLexer implements FlexLexer {
private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
private static final String ZZ_ATTRIBUTE_PACKED_0 =
"\3\0\1\11\6\1\1\0\1\11\1\0\3\1\1\11";
"\3\0\1\11\2\1\1\11\3\1\1\0\1\11\1\0"+
"\1\11";
private static int [] zzUnpackAttribute() {
int [] result = new int[17];
int [] result = new int[14];
int offset = 0;
offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
return result;
@@ -231,18 +231,6 @@ class _KDocLexer implements FlexLexer {
return CharArrayUtil.containLineBreaks(zzBuffer, zzStartRead, zzMarkedPos);
}
private final void pushbackEnd() {
if (isLastToken() && CharArrayUtil.regionMatches(zzBuffer, zzMarkedPos - 2, zzMarkedPos, "*/")) {
yypushback(2);
}
}
private final void pushbackText() {
int i = zzStartRead;
while (zzBuffer.charAt(i) == '*') i++;
yypushback(zzMarkedPos - i);
}
_KDocLexer(java.io.Reader in) {
this.zzReader = in;
@@ -497,8 +485,8 @@ class _KDocLexer implements FlexLexer {
}
case 7: break;
case 5:
{ if (isLastToken())
return KDocTokens.END;
{ if (isLastToken()) return KDocTokens.END;
else return KDocTokens.TEXT;
}
case 8: break;
case 1:
@@ -506,20 +494,23 @@ class _KDocLexer implements FlexLexer {
}
case 9: break;
case 2:
{ pushbackEnd();
return KDocTokens.TEXT;
{ yybegin(CONTENTS);
return KDocTokens.TEXT;
}
case 10: break;
case 4:
{ pushbackText();
yybegin(CONTENTS);
return KDocTokens.LEADING_ASTERISK;
{ yybegin(CONTENTS);
return KDocTokens.LEADING_ASTERISK;
}
case 11: break;
case 3:
{ if (yytextContainLineBreaks())
yybegin(LINE_BEGINNING);
return TokenType.WHITE_SPACE;
{ if (yytextContainLineBreaks()) {
yybegin(LINE_BEGINNING);
return TokenType.WHITE_SPACE;
} else {
yybegin(CONTENTS);
return KDocTokens.TEXT; // internal white space
}
}
case 12: break;
default:
compiler/frontend/src/org/jetbrains/jet/kdoc/psi/api/KDocElement.java
+2 -2
View File
@@ -16,7 +16,7 @@
package org.jetbrains.jet.kdoc.psi.api;
import org.jetbrains.jet.lang.psi.JetElement;
import com.intellij.psi.PsiElement;
public interface KDocElement extends JetElement {
public interface KDocElement extends PsiElement {
}
compiler/frontend/src/org/jetbrains/jet/lang/psi/JetPsiUtil.java
+15
View File
@@ -28,10 +28,12 @@ import com.intellij.psi.impl.CheckUtil;
import com.intellij.psi.impl.source.codeStyle.CodeEditUtil;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.util.codeInsight.CommentUtilCore;
import com.intellij.util.containers.ContainerUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jet.JetNodeTypes;
import org.jetbrains.jet.kdoc.psi.api.KDocElement;
import org.jetbrains.jet.lang.parsing.JetExpressionParsing;
import org.jetbrains.jet.lang.resolve.ImportPath;
import org.jetbrains.jet.lang.resolve.name.FqName;
@@ -740,4 +742,17 @@ public class JetPsiUtil {
public static String getText(@Nullable PsiElement element) {
return element != null ? element.getText() : "";
}
/**
* CommentUtilCore.isComment fails if element <strong>inside</strong> comment.
*
* Also, we can not add KDocTokens to COMMENTS TokenSet, because it is used in JetParserDefinition.getCommentTokens(),
* and therefor all COMMENTS tokens will be ignored by PsiBuilder.
*
* @param element
* @return
*/
public static boolean isInComment(PsiElement element) {
return CommentUtilCore.isComment(element) || element instanceof KDocElement;
}
}
compiler/frontend/src/org/jetbrains/jet/lexer/JetTokens.java
+8 -2
View File
@@ -184,8 +184,14 @@ public interface JetTokens {
PROTECTED_KEYWORD, OUT_KEYWORD, IN_KEYWORD, FINAL_KEYWORD, VARARG_KEYWORD, INLINE_KEYWORD, REIFIED_KEYWORD
);
TokenSet WHITESPACES = TokenSet.create(TokenType.WHITE_SPACE);
TokenSet COMMENTS = TokenSet.orSet(TokenSet.create(EOL_COMMENT, BLOCK_COMMENT, DOC_COMMENT, SHEBANG_COMMENT),
KDocTokens.ALL_KDOC_TOKENS);
/**
* Don't add KDocTokens to COMMENTS TokenSet, because it is used in JetParserDefinition.getCommentTokens(),
* and therefor all COMMENTS tokens will be ignored by PsiBuilder.
*
* @see org.jetbrains.jet.lang.psi.JetPsiUtil.isInComment()
*/
TokenSet COMMENTS = TokenSet.create(EOL_COMMENT, BLOCK_COMMENT, DOC_COMMENT, SHEBANG_COMMENT);
TokenSet WHITE_SPACE_OR_COMMENT_BIT_SET = TokenSet.orSet(COMMENTS, TokenSet.create(WHITE_SPACE));
TokenSet STRINGS = TokenSet.create(CHARACTER_LITERAL, REGULAR_STRING_PART);
compiler/frontend/src/org/jetbrains/jet/lexer/_JetLexer.java
+2 -2
View File
@@ -1,4 +1,4 @@
/* The following code was generated by JFlex 1.4.3 on 23.05.13 19:26 */
/* The following code was generated by JFlex 1.4.3 on 25.05.13 15:08 */
package org.jetbrains.jet.lexer;
@@ -11,7 +11,7 @@ import com.intellij.util.containers.Stack;
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.3
* on 23.05.13 19:26 from the specification file
* on 25.05.13 15:08 from the specification file
* <tt>/Users/factitious/Documents/kotlin/compiler/frontend/src/org/jetbrains/jet/lexer/Jet.flex</tt>
*/
class _JetLexer implements FlexLexer {
compiler/testData/psi/EOLsInComments.txt
+1 -1
View File
@@ -27,7 +27,7 @@ JetFile: EOLsInComments.jet
PsiWhiteSpace('\n ')
KDoc
PsiElement(KDOC_START)('/**')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)(' ')
PsiElement(KDOC_END)('*/')
PREFIX_EXPRESSION
OPERATION_REFERENCE
compiler/testData/psi/NestedComments.txt
+4 -7
View File
@@ -41,8 +41,7 @@ JetFile: NestedComments.jet
PsiWhiteSpace('\n')
KDoc
PsiElement(KDOC_START)('/**')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('/***/')
PsiElement(KDOC_TEXT)(' /***/')
PsiElement(KDOC_END)('*/')
PsiWhiteSpace('\n')
REFERENCE_EXPRESSION
@@ -50,12 +49,10 @@ JetFile: NestedComments.jet
PsiWhiteSpace('\n')
KDoc
PsiElement(KDOC_START)('/**')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('/**')
PsiElement(KDOC_TEXT)(' /**')
PsiWhiteSpace('\n\n')
PsiElement(KDOC_LEADING_ASTERISK)('*')
PsiElement(KDOC_TEXT)('/**')
PsiElement(KDOC_END)('*/')
PsiElement(KDOC_TEXT)('*/')
PsiElement(KDOC_END)('***/')
PsiWhiteSpace('\n')
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('b')
compiler/testData/psi/examples/array/MutableArray.txt
+1 -35
View File
@@ -2,41 +2,7 @@ JetFile: MutableArray.jet
KDoc
PsiElement(KDOC_START)('/**')
PsiWhiteSpace('\n ')
PsiElement(KDOC_TEXT)('These')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('declarations')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('are')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('"shallow"')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('in')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('the')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('sense')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('that')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('they')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('are')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('not')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('really')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('compiled,')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('only')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('the')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('type-checker')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('uses')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('them')
PsiElement(KDOC_TEXT)('These declarations are "shallow" in the sense that they are not really compiled, only the type-checker uses them')
PsiWhiteSpace('\n')
PsiElement(KDOC_END)('*/')
PsiWhiteSpace('\n\n')
compiler/testData/psi/kdoc/Simple.txt
+7 -36
View File
@@ -1,52 +1,23 @@
JetFile: Simple.kt
KDoc
PsiElement(KDOC_START)('/**')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('line')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('0')
PsiElement(KDOC_TEXT)(' line 0')
PsiWhiteSpace('\n ')
PsiElement(KDOC_TEXT)('line')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('1')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('//')
PsiElement(KDOC_TEXT)('line 1 //')
PsiWhiteSpace('\n ')
PsiElement(KDOC_LEADING_ASTERISK)('**')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('line')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('2')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('/*')
PsiElement(KDOC_TEXT)(' line 2 /*')
PsiWhiteSpace('\n ')
PsiElement(KDOC_TEXT)('line')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('3')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('/**')
PsiElement(KDOC_TEXT)('line 3 /**')
PsiWhiteSpace('\n')
PsiElement(KDOC_LEADING_ASTERISK)('*')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('line')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('*')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('4')
PsiElement(KDOC_TEXT)(' line * 4')
PsiWhiteSpace('\n ')
PsiElement(KDOC_LEADING_ASTERISK)('***')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('line')
PsiWhiteSpace(' */')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('5')
PsiElement(KDOC_TEXT)(' line */ 5')
PsiWhiteSpace('\n ')
PsiElement(KDOC_LEADING_ASTERISK)('**')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('line')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)('6')
PsiWhiteSpace(' ')
PsiElement(KDOC_TEXT)(' line 6 ')
PsiElement(KDOC_END)('*/')
NAMESPACE_HEADER
<empty list>
idea/src/org/jetbrains/jet/plugin/completion/JetKeywordCompletionContributor.java
+1 -2
View File
@@ -36,7 +36,6 @@ import com.intellij.psi.impl.source.tree.LeafPsiElement;
import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.util.ArrayUtil;
import com.intellij.util.ProcessingContext;
import com.intellij.util.codeInsight.CommentUtilCore;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jet.lang.psi.*;
import org.jetbrains.jet.lexer.JetToken;
@@ -103,7 +102,7 @@ public class JetKeywordCompletionContributor extends CompletionContributor {
return false;
}
return CommentUtilCore.isComment((PsiElement) element);
return JetPsiUtil.isInComment((PsiElement) element);
}
@Override
idea/src/org/jetbrains/jet/plugin/formatter/JetFormattingModelBuilder.java
-4
View File
@@ -25,7 +25,6 @@ import com.intellij.psi.codeStyle.CodeStyleSettings;
import com.intellij.psi.codeStyle.CommonCodeStyleSettings;
import com.intellij.psi.tree.TokenSet;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jet.kdoc.lexer.KDocTokens;
import org.jetbrains.jet.plugin.JetLanguage;
import static org.jetbrains.jet.JetNodeTypes.*;
@@ -108,9 +107,6 @@ public class JetFormattingModelBuilder implements FormattingModelBuilder {
.between(VALUE_ARGUMENT_LIST, FUNCTION_LITERAL_EXPRESSION).spaces(1)
.aroundInside(ARROW, WHEN_ENTRY).spaces(1)
// KDoc
.between(KDocTokens.LEADING_ASTERISK, KDocTokens.TEXT).spacing(1, 100, 0, true, 100)
;
}