Whitespace problems fixed

This commit is contained in:
Andrey Breslav
2011-01-01 03:02:29 +03:00
parent 4c41b1e566
commit 70293ca754
9 changed files with 458 additions and 81 deletions
@@ -140,13 +140,7 @@ import static org.jetbrains.jet.lexer.JetTokens.*;
}
protected IElementType lookahead(int k) {
// TODO: use a faster implementation
PsiBuilder.Marker tmp = mark();
for (int i = 0; i < k; i++) advance();
IElementType tt = tt();
tmp.rollbackTo();
return tt;
return myBuilder.lookAhead(k);
}
protected void consumeIf(JetToken token) {
@@ -21,7 +21,7 @@ public class JetExpressionParsing extends AbstractJetParsing {
NEW_KEYWORD, TRUE_KEYWORD, FALSE_KEYWORD, IS_KEYWORD, THROW_KEYWORD, RETURN_KEYWORD, BREAK_KEYWORD,
CONTINUE_KEYWORD, OBJECT_KEYWORD, IF_KEYWORD, TRY_KEYWORD, ELSE_KEYWORD, WHILE_KEYWORD, DO_KEYWORD,
MATCH_KEYWORD, RBRACKET, RBRACE, RPAR, PLUSPLUS, MINUSMINUS, MUL, PLUS, MINUS, EXCL, DIV, PERC, LTEQ,
// TODO GTEQ,
// TODO GTEQ, foo<bar, baz>=x
EQEQEQ, ARROW, DOUBLE_ARROW, EXCLEQEQEQ, EQEQ, EXCLEQ, ANDAND, OROR, SAFE_ACCESS, ELVIS, QUEST,
SEMICOLON, RANGE, EQ, MULTEQ, DIVEQ, PERCEQ, PLUSEQ, MINUSEQ, NOT_IN, NOT_IS, HASH, EOL_OR_SEMICOLON
);
@@ -194,7 +194,7 @@ public class JetExpressionParsing extends AbstractJetParsing {
parseValueArgumentList();
expression.done(CALL_EXPRESSION);
} else if (at(LT)) {
// TODO: be more clever: f(foo<a, (a + 1), b>(c)) is not a function call
// TODO: be (even) more clever
int gtPos = matchTokenStreamPredicate(new FirstBefore(new At(GT), new AtSet(TYPE_ARGUMENT_LIST_STOPPERS, false)) {
@Override
public boolean isTopLevel(int openAngleBrackets, int openBrackets, int openBraces, int openParentheses) {
@@ -58,15 +58,15 @@ public class PsiBuilderAdapter implements PsiBuilder {
return myDelegate.lookAhead(steps);
}
// @Override
// public IElementType rawLookup(int steps) {
// return myDelegate.rawLookup(steps);
// }
//
// @Override
// public int rawTokenTypeStart(int steps) {
// return myDelegate.rawTokenTypeStart(steps);
// }
@Override
public IElementType rawLookup(int steps) {
return myDelegate.rawLookup(steps);
}
@Override
public int rawTokenTypeStart(int steps) {
return myDelegate.rawTokenTypeStart(steps);
}
@Nullable
@NonNls
@@ -2,7 +2,7 @@ package org.jetbrains.jet.lang.parsing;
import com.intellij.lang.PsiBuilder;
import com.intellij.lang.WhitespaceSkippedCallback;
import com.intellij.psi.TokenType;
import com.intellij.psi.tree.IElementType;
import java.util.Stack;
@@ -11,77 +11,44 @@ import java.util.Stack;
* @author abreslav
*/
public class SemanticWhitespaceAwarePsiBuilderImpl extends PsiBuilderAdapter implements SemanticWhitespaceAwarePsiBuilder {
private final WhitespaceSkippedCallback myWhitespaceSkippedCallback = new WhitespaceSkippedCallback() {
public void onSkip(IElementType type, int start, int end) {
CharSequence whitespace = getOriginalText();
for (int i = start; i < end; i++) {
char c = whitespace.charAt(i);
if (c == '\n') {
myEOLInLastWhitespace = true;
break;
}
}
}
};
private boolean myEOLInLastWhitespace;
private final Stack<Boolean> myEOLsEnabled = new Stack<Boolean>();
private final Stack<Boolean> eolsEnabled = new Stack<Boolean>();
public SemanticWhitespaceAwarePsiBuilderImpl(final PsiBuilder delegate) {
super(delegate);
delegate.setWhitespaceSkippedCallback(myWhitespaceSkippedCallback);
myEOLsEnabled.push(true);
eolsEnabled.push(true);
}
@Override
public void advanceLexer() {
myEOLInLastWhitespace = false;
super.advanceLexer();
}
@Override
public boolean eolInLastWhitespace() {
return myEOLsEnabled.peek() && (myEOLInLastWhitespace || eof());
if (!eolsEnabled.peek()) return false;
if (eof()) return true;
IElementType previousToken = rawLookup(-1);
if (previousToken == TokenType.WHITE_SPACE) {
int previousTokenStart = rawTokenTypeStart(-1);
int previousTokenEnd = rawTokenTypeStart(0);
assert previousTokenStart >= 0;
assert previousTokenEnd < getOriginalText().length();
for (int i = previousTokenStart; i < previousTokenEnd; i++) {
if (getOriginalText().charAt(i) == '\n') return true;
}
}
return false;
}
@Override
public void disableEols() {
myEOLsEnabled.push(false);
eolsEnabled.push(false);
}
@Override
public void enableEols() {
myEOLsEnabled.push(true);
eolsEnabled.push(true);
}
@Override
public void restoreEolsState() {
assert myEOLsEnabled.size() > 1;
myEOLsEnabled.pop();
assert eolsEnabled.size() > 1;
eolsEnabled.pop();
}
// TODO: Overhead
@Override
public Marker mark() {
return new MarkerWrapper(super.mark());
}
private class MarkerWrapper extends MarkerAdapter {
private final boolean eolInLastWhitespace = eolInLastWhitespace();
public MarkerWrapper(Marker delegate) {
super(delegate);
}
@Override
public void rollbackTo() {
super.rollbackTo();
myEOLInLastWhitespace = eolInLastWhitespace;
}
@Override
public Marker precede() {
return new MarkerWrapper(super.precede());
}
}
}
+4 -4
View File
@@ -184,8 +184,8 @@ JetFile: ControlStructures.jet
PsiWhiteSpace('\n ')
BREAK
PsiElement(break)('break')
PsiWhiteSpace('\n ')
PsiElement(IDENTIFIER)('la')
PsiWhiteSpace('\n ')
PsiElement(IDENTIFIER)('la')
PsiWhiteSpace('\n ')
BREAK
PsiElement(break)('break')
@@ -194,8 +194,8 @@ JetFile: ControlStructures.jet
PsiWhiteSpace('\n ')
CONTINUE
PsiElement(continue)('continue')
PsiWhiteSpace('\n ')
PsiElement(IDENTIFIER)('la')
PsiWhiteSpace('\n ')
PsiElement(IDENTIFIER)('la')
PsiWhiteSpace('\n ')
CONTINUE
PsiElement(continue)('continue')
+16
View File
@@ -86,5 +86,21 @@ JetFile: EOLsOnRollback.jet
PsiWhiteSpace(' ')
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('4')
PsiWhiteSpace('\n\n ')
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('1')
PsiWhiteSpace('\n ')
PROPERTY
MODIFIER_LIST
ATTRIBUTE_ANNOTATION
PsiElement(LBRACKET)('[')
ATTRIBUTE
USER_TYPE
PsiElement(IDENTIFIER)('a')
PsiElement(RBRACKET)(']')
PsiWhiteSpace(' ')
PsiElement(val)('val')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('f')
PsiWhiteSpace('\n')
PsiElement(RBRACE)('}')
+400
View File
@@ -0,0 +1,400 @@
JetFile: Precedence.jet
NAMESPACE
FUN
PsiElement(fun)('fun')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('foo')
TYPE_PARAMETER_LIST
<empty list>
VALUE_PARAMETER_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
PsiWhiteSpace(' ')
BLOCK
PsiElement(LBRACE)('{')
PsiWhiteSpace('\n ')
DOT_QIALIFIED_EXPRESSION
CALL_EXPRESSION
PsiElement(IDENTIFIER)('b')
VALUE_ARGUMENT_LIST
PsiElement(LPAR)('(')
PsiElement(RPAR)(')')
PsiElement(DOT)('.')
PsiElement(IDENTIFIER)('x')
PsiWhiteSpace('\n ')
DOT_QIALIFIED_EXPRESSION
POSTFIX_EXPRESSION
PsiElement(IDENTIFIER)('x')
PsiElement(PLUSPLUS)('++')
PsiWhiteSpace(' ')
PsiElement(DOT)('.')
PsiWhiteSpace(' ')
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('4')
PsiWhiteSpace('\n ')
PREFIX_EXPRESSION
PsiElement(PLUSPLUS)('++')
PsiWhiteSpace(' ')
PREFIX_EXPRESSION
PsiElement(MINUSMINUS)('--')
PsiWhiteSpace(' ')
PREFIX_EXPRESSION
PsiElement(EXCL)('!')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace('\n ')
CALL_EXPRESSION
PsiElement(IDENTIFIER)('f')
VALUE_ARGUMENT_LIST
PsiElement(LPAR)('(')
VALUE_ARGUMENT
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('foo')
PsiElement(LT)('<')
PsiElement(IDENTIFIER)('a')
PsiElement(COMMA)(',')
PsiWhiteSpace(' ')
VALUE_ARGUMENT
TUPLE
PsiElement(LPAR)('(')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('b')
PsiElement(GT)('>')
TUPLE
PsiElement(LPAR)('(')
PsiElement(IDENTIFIER)('x')
PsiElement(RPAR)(')')
PsiElement(RPAR)(')')
PsiElement(RPAR)(')')
PsiWhiteSpace('\n ')
CALL_EXPRESSION
PsiElement(IDENTIFIER)('f')
VALUE_ARGUMENT_LIST
PsiElement(LPAR)('(')
VALUE_ARGUMENT
CALL_EXPRESSION
PsiElement(IDENTIFIER)('foo')
TYPE_ARGUMENT_LIST
PsiElement(LT)('<')
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('a')
PsiElement(COMMA)(',')
PsiWhiteSpace(' ')
TYPE_REFERENCE
USER_TYPE
PsiElement(IDENTIFIER)('b')
PsiElement(GT)('>')
VALUE_ARGUMENT_LIST
PsiElement(LPAR)('(')
VALUE_ARGUMENT
PsiElement(IDENTIFIER)('x')
PsiElement(RPAR)(')')
PsiElement(RPAR)(')')
PsiWhiteSpace('\n ')
CALL_EXPRESSION
PsiElement(IDENTIFIER)('f')
VALUE_ARGUMENT_LIST
PsiElement(LPAR)('(')
VALUE_ARGUMENT
TUPLE
PsiElement(LPAR)('(')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('foo')
PsiElement(LT)('<')
PsiElement(IDENTIFIER)('a')
PsiElement(RPAR)(')')
PsiElement(COMMA)(',')
PsiWhiteSpace(' ')
VALUE_ARGUMENT
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('b')
PsiElement(GT)('>')
TUPLE
PsiElement(LPAR)('(')
PsiElement(IDENTIFIER)('x')
PsiElement(RPAR)(')')
PsiElement(RPAR)(')')
PsiWhiteSpace('\n ')
CALL_EXPRESSION
PsiElement(IDENTIFIER)('f')
VALUE_ARGUMENT_LIST
PsiElement(LPAR)('(')
VALUE_ARGUMENT
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('foo')
PsiElement(LT)('<')
PsiElement(IDENTIFIER)('a')
PsiElement(COMMA)(',')
PsiWhiteSpace(' ')
VALUE_ARGUMENT
PsiElement(IDENTIFIER)('b')
PsiElement(RPAR)(')')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(PLUS)('+')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(PLUS)('+')
PsiWhiteSpace(' ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(MUL)('*')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(PLUS)('+')
PsiWhiteSpace(' ')
TUPLE
PsiElement(LPAR)('(')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(MUL)('*')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('c')
PsiElement(RPAR)(')')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(PLUS)('+')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(GT)('>')
PsiWhiteSpace(' ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace(' ')
PsiElement(MUL)('*')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(PLUS)('+')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(in)('in')
PsiWhiteSpace(' ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace(' ')
PsiElement(MUL)('*')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('d')
PsiWhiteSpace('\n ')
DOT_QIALIFIED_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiElement(DOT)('.')
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(PLUS)('+')
PsiWhiteSpace(' ')
DOT_QIALIFIED_EXPRESSION
PsiElement(IDENTIFIER)('b')
PsiElement(DOT)('.')
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
DOT_QIALIFIED_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiElement(DOT)('.')
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(PLUS)('+')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace('\n ')
POSTFIX_EXPRESSION
DOT_QIALIFIED_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiElement(DOT)('.')
PsiElement(IDENTIFIER)('b')
PsiElement(PLUSPLUS)('++')
PsiWhiteSpace('\n ')
PREFIX_EXPRESSION
PsiElement(MINUSMINUS)('--')
POSTFIX_EXPRESSION
DOT_QIALIFIED_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiElement(DOT)('.')
PsiElement(IDENTIFIER)('b')
PsiElement(PLUSPLUS)('++')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
PREFIX_EXPRESSION
PsiElement(MINUSMINUS)('--')
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(MUL)('*')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiElement(PLUS)('+')
PsiElement(IDENTIFIER)('b')
PsiElement(RANGE)('..')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('b')
PsiElement(MINUS)('-')
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('1')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
BINARY_EXPRESSION
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('1')
PsiElement(RANGE)('..')
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('2')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('foo')
PsiWhiteSpace(' ')
BINARY_EXPRESSION
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('2')
PsiElement(RANGE)('..')
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('3')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
BINARY_EXPRESSION
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('1')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('foo')
PsiWhiteSpace(' ')
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('2')
PsiWhiteSpace(' ')
PsiElement(ELVIS)('?:')
PsiWhiteSpace(' ')
BINARY_EXPRESSION
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('1')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('bar')
PsiWhiteSpace(' ')
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('3')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(ELVIS)('?:')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(in)('in')
PsiWhiteSpace(' ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('b')
PsiElement(ELVIS)('?:')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(COLON)(':')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(LT)('<')
PsiWhiteSpace(' ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(COLON)(':')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(LT)('<')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(EQEQ)('==')
PsiWhiteSpace(' ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(GT)('>')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(EXCLEQ)('!=')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(ANDAND)('&&')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(OROR)('||')
PsiWhiteSpace(' ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(ANDAND)('&&')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(EQ)('=')
PsiWhiteSpace(' ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(ARROW)('->')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace('\n ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('a')
PsiWhiteSpace(' ')
PsiElement(EQ)('=')
PsiWhiteSpace(' ')
BINARY_EXPRESSION
PsiElement(IDENTIFIER)('b')
PsiWhiteSpace(' ')
PsiElement(OROR)('||')
PsiWhiteSpace(' ')
PsiElement(IDENTIFIER)('c')
PsiWhiteSpace('\n')
PsiElement(RBRACE)('}')
+2 -2
View File
@@ -7,8 +7,8 @@ JetFile: Properties.jet
PsiWhiteSpace(' ')
PsiElement(EQ)('=')
PsiWhiteSpace(' ')
BINARY_EXPRESSION
BINARY_EXPRESSION
DOT_QIALIFIED_EXPRESSION
DOT_QIALIFIED_EXPRESSION
PsiElement(IDENTIFIER)('bar')
PsiElement(DOT)('.')
PsiElement(IDENTIFIER)('foo')
+4 -4
View File
@@ -605,8 +605,8 @@ JetFile: SimpleExpressions.jet
PsiWhiteSpace('\n ')
BREAK
PsiElement(break)('break')
PsiWhiteSpace('\n ')
PsiElement(IDENTIFIER)('la')
PsiWhiteSpace('\n ')
PsiElement(IDENTIFIER)('la')
PsiWhiteSpace('\n ')
BREAK
PsiElement(break)('break')
@@ -615,8 +615,8 @@ JetFile: SimpleExpressions.jet
PsiWhiteSpace('\n ')
CONTINUE
PsiElement(continue)('continue')
PsiWhiteSpace('\n ')
PsiElement(IDENTIFIER)('la')
PsiWhiteSpace('\n ')
PsiElement(IDENTIFIER)('la')
PsiWhiteSpace('\n ')
CONTINUE
PsiElement(continue)('continue')