azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Token stream predicates for lookahead

Browse Source
This commit is contained in:
Andrey Breslav
2010-12-29 16:56:44 +03:00
parent 9d9ee762e1
commit 88b5c26fe3
11 changed files with 317 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
idea/src/org/jetbrains/jet/lang/parsing/AbstractJetParsing.java
+58 -23
View File
@@ -155,35 +155,20 @@ import static org.jetbrains.jet.lexer.JetTokens.*;
error.error(mesage);
}
/*
* Looks for a the last top-level (not inside any {} [] () <>) '.' occurring before a
* top-level occurrence of a token from the <code>stopSet</code>
*
* Returns -1 if no occurrence is found
*/
protected int findLastBefore(TokenSet lookFor, TokenSet stopAt, boolean dontStopRightAfterOccurrence) {
protected int matchTokenStreamPredicate(TokenStreamPattern pattern) {
PsiBuilder.Marker currentPosition = mark();
int lastOccurrence = -1;
int openAngleBrackets = 0;
int openBraces = 0;
int openParentheses = 0;
int openBrackets = 0;
IElementType previousToken = null;
while (!eof()) {
if (atSet(lookFor)
&& openAngleBrackets == 0
if (pattern.processToken(
myBuilder.getCurrentOffset(),
openAngleBrackets == 0
&& openBrackets == 0
&& openBraces == 0
&& openParentheses == 0) {
lastOccurrence = myBuilder.getCurrentOffset();
}
if (atSet(stopAt)) {
if (openAngleBrackets == 0
&& openBrackets == 0
&& openBraces == 0
&& openParentheses == 0
&& (!dontStopRightAfterOccurrence
|| !lookFor.contains(previousToken))) break;
&& openParentheses == 0)) {
break;
}
if (at(LPAR)) {
openParentheses++;
@@ -209,15 +194,65 @@ import static org.jetbrains.jet.lexer.JetTokens.*;
else if (at(RBRACKET)) {
openBrackets--;
}
previousToken = tt();
advance(); // skip token
}
currentPosition.rollbackTo();
return lastOccurrence;
return pattern.result();
}
/*
* Looks for a the last top-level (not inside any {} [] () <>) '.' occurring before a
* top-level occurrence of a token from the <code>stopSet</code>
*
* Returns -1 if no occurrence is found
*/
protected int findLastBefore(TokenSet lookFor, TokenSet stopAt, boolean dontStopRightAfterOccurrence) {
return matchTokenStreamPredicate(new LastBefore(new AtSet(lookFor), new AtSet(stopAt), dontStopRightAfterOccurrence));
}
protected boolean eol() {
return myBuilder.eolInLastWhitespace() || eof();
}
protected class At implements TokenStreamPredicate {
private final IElementType lookFor;
private final boolean topLevelOnly;
public At(IElementType lookFor, boolean topLevelOnly) {
this.lookFor = lookFor;
this.topLevelOnly = topLevelOnly;
}
public At(IElementType lookFor) {
this(lookFor, true);
}
@Override
public boolean matching(boolean topLevel) {
return (topLevel || !topLevelOnly) && at(lookFor);
}
}
protected class AtSet implements TokenStreamPredicate {
private final TokenSet lookFor;
private final boolean topLevelOnly;
public AtSet(TokenSet lookFor, boolean topLevelOnly) {
this.lookFor = lookFor;
this.topLevelOnly = topLevelOnly;
}
public AtSet(TokenSet lookFor) {
this(lookFor, true);
}
@Override
public boolean matching(boolean topLevel) {
return (topLevel || !topLevelOnly) && atSet(lookFor);
}
}
}
idea/src/org/jetbrains/jet/lang/parsing/AbstractTokenStreamPattern.java
+15
View File
@@ -0,0 +1,15 @@
package org.jetbrains.jet.lang.parsing;
/**
* @author abreslav
*/
public abstract class AbstractTokenStreamPattern implements TokenStreamPattern {
protected int lastOccurrence = -1;
@Override
public int result() {
return lastOccurrence;
}
}
idea/src/org/jetbrains/jet/lang/parsing/FirstBefore.java
+26
View File
@@ -0,0 +1,26 @@
package org.jetbrains.jet.lang.parsing;
/**
* @author abreslav
*/
public class FirstBefore extends AbstractTokenStreamPattern {
private final TokenStreamPredicate lookFor;
private final TokenStreamPredicate stopAt;
public FirstBefore(TokenStreamPredicate lookFor, TokenStreamPredicate stopAt) {
this.lookFor = lookFor;
this.stopAt = stopAt;
}
@Override
public boolean processToken(int offset, boolean topLevel) {
if (lookFor.matching(topLevel)) {
lastOccurrence = offset;
return true;
}
if (stopAt.matching(topLevel)) {
return true;
}
return false;
}
}
idea/src/org/jetbrains/jet/lang/parsing/JetParsing.java
+20 -2
View File
@@ -694,9 +694,26 @@ public class JetParsing extends AbstractJetParsing {
advance(); // VAL_KEYWORD or VAR_KEYWORD
// TODO: EOL_OR_SEMICOLON is too restrictive here
TokenSet propertyNameFollow = TokenSet.create(COLON, EQ, LBRACE, SEMICOLON, EOL_OR_SEMICOLON);
TokenSet propertyNameFollow = TokenSet.create(COLON, EQ, LBRACE, SEMICOLON);
int lastDot = findLastBefore(TokenSet.create(DOT), propertyNameFollow, true);
// int lastDot = findLastBefore(TokenSet.create(DOT), propertyNameFollow, true);
// TODO: constant
int lastDot = matchTokenStreamPredicate(new FirstBefore(new TokenStreamPredicate() {
@Override
public boolean matching(boolean topLevel) {
return topLevel
&& at(DOT);
}
}, new TokenStreamPredicate() {
@Override
public boolean matching(boolean topLevel) {
if (lookahead(1) == IDENTIFIER) {
IElementType lookahead2 = lookahead(2);
return lookahead2 != LT && lookahead2 != DOT;
}
return false;
}
}));
if (lastDot == -1) {
parseAttributeList();
@@ -1258,6 +1275,7 @@ public class JetParsing extends AbstractJetParsing {
int lastLPar = findLastBefore(TokenSet.create(LPAR), TokenSet.create(RBRACE, COLON), false);
if (lastLPar >= 0 && lastLPar > myBuilder.getCurrentOffset()) {
PsiBuilder.Marker receiverType = mark();
// TODO : -1 is a hack
createTruncatedBuilder(lastLPar - 1).parseTypeRef();
receiverType.done(RECEIVER_TYPE);
advance(); // DOT
idea/src/org/jetbrains/jet/lang/parsing/LastBefore.java
+37
View File
@@ -0,0 +1,37 @@
package org.jetbrains.jet.lang.parsing;
/**
* @author abreslav
*/
public class LastBefore extends AbstractTokenStreamPattern {
private final boolean dontStopRightAfterOccurrence;
private final TokenStreamPredicate lookFor;
private final TokenStreamPredicate stopAt;
private boolean previousLookForResult;
public LastBefore(TokenStreamPredicate lookFor, TokenStreamPredicate stopAt, boolean dontStopRightAfterOccurrence) {
this.lookFor = lookFor;
this.stopAt = stopAt;
this.dontStopRightAfterOccurrence = dontStopRightAfterOccurrence;
}
public LastBefore(TokenStreamPredicate lookFor, TokenStreamPredicate stopAt) {
this(lookFor, stopAt, false);
}
@Override
public boolean processToken(int offset, boolean topLevel) {
boolean lookForResult = lookFor.matching(topLevel);
if (lookForResult) {
lastOccurrence = offset;
}
if (stopAt.matching(topLevel)) {
if (topLevel
&& (!dontStopRightAfterOccurrence
|| !previousLookForResult)) return true;
}
previousLookForResult = lookForResult;
return false;
}
}
idea/src/org/jetbrains/jet/lang/parsing/TokenStreamPattern.java
+22
View File
@@ -0,0 +1,22 @@
package org.jetbrains.jet.lang.parsing;
/**
* @author abreslav
*/
public interface TokenStreamPattern {
/**
* Called on each token
*
*
*
* @param offset
* @param topLevel indicates if no brackets (of types () [] {} <>) are currently unmatched
* @return <code>true</code> to stop
*/
boolean processToken(int offset, boolean topLevel);
/**
* @return the position where the predicate has matched, -1 if no match was found
*/
int result();
}
idea/src/org/jetbrains/jet/lang/parsing/TokenStreamPredicate.java
+8
View File
@@ -0,0 +1,8 @@
package org.jetbrains.jet.lang.parsing;
/**
* @author abreslav
*/
public interface TokenStreamPredicate {
boolean matching(boolean topLevel);
}
idea/testData/psi/Properties.jet
+8 -6
View File
@@ -1,7 +1,9 @@
val foo
val [a] foo
val foo.bar
val foo.bar.{() : ()}.foo
val {() : ()}.foo
val {foo.bar.() : ()}.foo
val foo : T
val [a] foo = bar
@@ -9,24 +11,24 @@ val foo.bar
get() {}
set(sad) = foo
val foo.bar.{() : ()}.foo = foo
val {foo.bar.() : ()}.foo = foo
get() {}
set(it) {}
val foo.bar.{() : ()}.foo = foo
val {foo.bar.() : ()}.foo = foo
get() : Foo {}
set(it) {}
val foo.bar.{() : ()}.foo : bar = foo
val {foo.bar.() : ()}.foo : bar = foo
[a] public get() {}
virtual set(a : b) {}
val foo.bar.{() : ()}.foo : bar = foo
val {foo.bar.() : ()}.foo : bar = foo
virtual set(a : b) {}
val foo.bar.{() : ()}.foo : bar = foo
val {foo.bar.() : ()}.foo : bar = foo
[a] public get() {}
idea/testData/psi/Properties.txt
+76 -48
View File
@@ -11,9 +11,8 @@ JetFile: Properties.jet
ATTRIBUTE_ANNOTATION
PsiElement(LBRACKET)('[')
ATTRIBUTE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('a')
USER_TYPE
PsiElement(IDENTIFIER)('a')
PsiElement(RBRACKET)(']')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('foo')
@@ -26,19 +25,41 @@ JetFile: Properties.jet
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
PsiElement(IDENTIFIER)('bar')
PsiWhiteSpace('\n\n')
PROPERTY
PsiElement(val)('val')
PsiWhiteSpace(' ')
TYPE_REFERENCE
FUNCTION_TYPE
PsiElement(LBRACE)('{')
VALUE_PARAMETER_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
PsiWhiteSpace(' ')
PsiElement(COLON)(':')
PsiWhiteSpace(' ')
TYPE_REFERENCE
TUPLE_TYPE
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
PsiElement(RBRACE)('}')
PsiElement(DOT)('.')
PsiElement(IDENTIFIER)('foo')
PsiWhiteSpace('\n')
PROPERTY
PsiElement(val)('val')
PsiWhiteSpace(' ')
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
FUNCTION_TYPE
PsiElement(LBRACE)('{')
RECEIVER_TYPE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
VALUE_PARAMETER_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
@@ -70,9 +91,8 @@ JetFile: Properties.jet
ATTRIBUTE_ANNOTATION
PsiElement(LBRACKET)('[')
ATTRIBUTE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('a')
USER_TYPE
PsiElement(IDENTIFIER)('a')
PsiElement(RBRACKET)(']')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('foo')
@@ -114,14 +134,16 @@ JetFile: Properties.jet
PsiElement(val)('val')
PsiWhiteSpace(' ')
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
FUNCTION_TYPE
PsiElement(LBRACE)('{')
RECEIVER_TYPE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
VALUE_PARAMETER_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
@@ -164,14 +186,16 @@ JetFile: Properties.jet
PsiElement(val)('val')
PsiWhiteSpace(' ')
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
FUNCTION_TYPE
PsiElement(LBRACE)('{')
RECEIVER_TYPE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
VALUE_PARAMETER_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
@@ -220,14 +244,16 @@ JetFile: Properties.jet
PsiElement(val)('val')
PsiWhiteSpace(' ')
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
FUNCTION_TYPE
PsiElement(LBRACE)('{')
RECEIVER_TYPE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
VALUE_PARAMETER_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
@@ -257,9 +283,8 @@ JetFile: Properties.jet
ATTRIBUTE_ANNOTATION
PsiElement(LBRACKET)('[')
ATTRIBUTE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('a')
USER_TYPE
PsiElement(IDENTIFIER)('a')
PsiElement(RBRACKET)(']')
PsiWhiteSpace(' ')
PsiElement(public)('public')
@@ -296,14 +321,16 @@ JetFile: Properties.jet
PsiElement(val)('val')
PsiWhiteSpace(' ')
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
FUNCTION_TYPE
PsiElement(LBRACE)('{')
RECEIVER_TYPE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
VALUE_PARAMETER_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
@@ -352,14 +379,16 @@ JetFile: Properties.jet
PsiElement(val)('val')
PsiWhiteSpace(' ')
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
FUNCTION_TYPE
PsiElement(LBRACE)('{')
RECEIVER_TYPE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
VALUE_PARAMETER_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
@@ -389,9 +418,8 @@ JetFile: Properties.jet
ATTRIBUTE_ANNOTATION
PsiElement(LBRACKET)('[')
ATTRIBUTE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('a')
USER_TYPE
PsiElement(IDENTIFIER)('a')
PsiElement(RBRACKET)(']')
PsiWhiteSpace(' ')
PsiElement(public)('public')
idea/testData/psi/Properties_ERR.jet
+3 -3
View File
@@ -9,11 +9,11 @@ val foo.bar
public () {}
() = foo
val foo.bar.{() : ()}.foo = foo
dfget() {}
val {foo.bar.() : ()}.foo = foo
set) {}
dfget() {}
val foo.bar.{() : ()}.foo = foo
val {foo.bar.() : ()}.foo = foo
get(foo) {}
set() {}
set() {}
idea/testData/psi/Properties_ERR.txt
+44 -40
View File
@@ -47,9 +47,8 @@ JetFile: Properties_ERR.jet
ATTRIBUTE_ANNOTATION
PsiElement(LBRACKET)('[')
ATTRIBUTE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('a')
USER_TYPE
PsiElement(IDENTIFIER)('a')
PsiErrorElement:Expecting ']' to close an attribute annotation
<empty list>
PsiWhiteSpace(' ')
@@ -65,12 +64,10 @@ JetFile: Properties_ERR.jet
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
PsiErrorElement:Expecting property name
<empty list>
PsiElement(IDENTIFIER)('bar')
PsiErrorElement:Expecting namespace or top level declaration
PsiElement(DOT)('.')
PsiWhiteSpace('\n')
PROPERTY
PsiElement(val)('val')
@@ -78,9 +75,8 @@ JetFile: Properties_ERR.jet
ATTRIBUTE_ANNOTATION
PsiElement(LBRACKET)('[')
ATTRIBUTE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('a')
USER_TYPE
PsiElement(IDENTIFIER)('a')
PsiElement(RBRACKET)(']')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('foo')
@@ -131,14 +127,16 @@ JetFile: Properties_ERR.jet
PsiElement(val)('val')
PsiWhiteSpace(' ')
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
FUNCTION_TYPE
PsiElement(LBRACE)('{')
RECEIVER_TYPE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
VALUE_PARAMETER_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
@@ -156,6 +154,18 @@ JetFile: Properties_ERR.jet
PsiElement(EQ)('=')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('foo')
PsiWhiteSpace('\n ')
PROPERTY_ACCESSOR
PsiElement(set)('set')
PsiErrorElement:Expecting '('
<empty list>
VALUE_PARAMETER
<empty list>
PsiElement(RPAR)(')')
PsiWhiteSpace(' ')
BLOCK
PsiElement(LBRACE)('{')
PsiElement(RBRACE)('}')
PsiWhiteSpace('\n ')
PsiErrorElement:Expecting namespace or top level declaration
PsiElement(IDENTIFIER)('dfget')
@@ -164,16 +174,6 @@ JetFile: Properties_ERR.jet
PsiErrorElement:Expecting namespace or top level declaration
PsiElement(RPAR)(')')
PsiWhiteSpace(' ')
PsiErrorElement:Expecting namespace or top level declaration
PsiElement(LBRACE)('{')
PsiErrorElement:Expecting namespace or top level declaration
PsiElement(RBRACE)('}')
PsiWhiteSpace('\n ')
PsiErrorElement:Expecting namespace or top level declaration
PsiElement(IDENTIFIER)('set')
PsiErrorElement:Expecting namespace or top level declaration
PsiElement(RPAR)(')')
PsiWhiteSpace(' ')
PsiErrorElement:Expecting namespace or top level declaration
PsiElement(LBRACE)('{')
PsiErrorElement:Expecting namespace or top level declaration
@@ -183,14 +183,16 @@ JetFile: Properties_ERR.jet
PsiElement(val)('val')
PsiWhiteSpace(' ')
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
FUNCTION_TYPE
PsiElement(LBRACE)('{')
RECEIVER_TYPE
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('foo')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
VALUE_PARAMETER_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
@@ -250,15 +252,17 @@ JetFile: Properties_ERR.jet
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('f')
PsiElement(DOT)('.')
USER_TYPE
PsiElement(IDENTIFIER)('d')
PsiElement(DOT)('.')
PsiErrorElement:Expecting property name
PsiElement(MINUS)('-')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('d')
PsiErrorElement:Expecting namespace or top level declaration
PsiElement(DOT)('.')
PsiErrorElement:Expecting namespace or top level declaration
PsiElement(MINUS)('-')
PsiWhiteSpace(' ')
PsiErrorElement:Expecting namespace or top level declaration
PsiElement(EQ)('=')
PsiWhiteSpace(' ')
PsiWhiteSpace(' ')
PsiErrorElement:Expecting namespace or top level declaration
PsiElement(IDENTIFIER)('f')
PsiWhiteSpace('\n\n')
PROPERTY