From ddf0213e44a2a2b646ed1de406c8f1dde04edeb3 Mon Sep 17 00:00:00 2001 From: Andrey Breslav Date: Thu, 16 Dec 2010 19:02:15 +0300 Subject: [PATCH] Expression parser factored out --- .idea/misc.xml | 2 +- .../jet/lang/parsing/AbstractJetParsing.java | 107 +++++++++++++ .../lang/parsing/JetExpressionParsing.java | 39 +++++ .../jetbrains/jet/lang/parsing/JetParser.java | 2 +- .../jet/lang/parsing/JetParsing.java | 147 +----------------- .../jet/lang/parsing/PsiBuilderAdapter.java | 122 +++++++++++++++ .../SemanticWitespaceAwarePsiBuilder.java | 41 +++++ 7 files changed, 319 insertions(+), 141 deletions(-) create mode 100644 idea/src/org/jetbrains/jet/lang/parsing/AbstractJetParsing.java create mode 100644 idea/src/org/jetbrains/jet/lang/parsing/JetExpressionParsing.java create mode 100644 idea/src/org/jetbrains/jet/lang/parsing/PsiBuilderAdapter.java create mode 100644 idea/src/org/jetbrains/jet/lang/parsing/SemanticWitespaceAwarePsiBuilder.java diff --git a/.idea/misc.xml b/.idea/misc.xml index 172449fee78..0f7b991f158 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -24,7 +24,7 @@ http://www.w3.org/1999/xhtml - + diff --git a/idea/src/org/jetbrains/jet/lang/parsing/AbstractJetParsing.java b/idea/src/org/jetbrains/jet/lang/parsing/AbstractJetParsing.java new file mode 100644 index 00000000000..815ec4e0fcd --- /dev/null +++ b/idea/src/org/jetbrains/jet/lang/parsing/AbstractJetParsing.java @@ -0,0 +1,107 @@ +package org.jetbrains.jet.lang.parsing; + +import com.intellij.lang.PsiBuilder; +import com.intellij.psi.tree.IElementType; +import com.intellij.psi.tree.TokenSet; +import org.jetbrains.jet.lexer.JetKeywordToken; +import org.jetbrains.jet.lexer.JetToken; + +import static org.jetbrains.jet.lexer.JetTokens.*; + +/** + * Created by IntelliJ IDEA. + * User: user + * Date: 12/16/10 + * Time: 6:34 PM + * To change this template use File | Settings | File Templates. + */ +/*package*/ class AbstractJetParsing { + protected final SemanticWitespaceAwarePsiBuilder myBuilder; + + public AbstractJetParsing(SemanticWitespaceAwarePsiBuilder builder) { + this.myBuilder = builder; + } + + protected boolean expect(JetToken expectation, String message) { + return expect(expectation, message, null); + } + + protected PsiBuilder.Marker mark() { + return myBuilder.mark(); + } + + protected void error(String message) { + myBuilder.error(message); + } + + protected boolean expect(JetToken expectation, String message, TokenSet recoverySet) { + if (at(expectation)) { + advance(); // expectation + return true; + } + + errorWithRecovery(message, recoverySet); + + return false; + } + + protected void errorWithRecovery(String message, TokenSet recoverySet) { + JetToken tt = tt(); + if (recoverySet == null || recoverySet.contains(tt) + || (recoverySet.contains(EOL_OR_SEMICOLON) + && (tt == SEMICOLON || myBuilder.eolInLastWhitespace()))) { + error(message); + } + else { + errorAndAdvance(message); + } + } + + protected void errorAndAdvance(String message) { + PsiBuilder.Marker err = mark(); + advance(); // erroneous token + err.error(message); + } + + protected boolean eof() { + return myBuilder.eof(); + } + + protected void advance() { + myBuilder.advanceLexer(); + } + + protected JetToken tt() { + return (JetToken) myBuilder.getTokenType(); + } + + protected boolean at(final IElementType expectation) { + JetToken token = tt(); + if (token == expectation) return true; + if (expectation == EOL_OR_SEMICOLON) { + if (token == SEMICOLON) return true; + if (myBuilder.eolInLastWhitespace()) return true; + } + if (token == IDENTIFIER && expectation instanceof JetKeywordToken) { + JetKeywordToken expectedKeyword = (JetKeywordToken) expectation; + if (expectedKeyword.isSoft() && expectedKeyword.getValue().equals(myBuilder.getTokenText())) { + myBuilder.remapCurrentToken(expectation); + return true; + } + } + return false; + } + + protected IElementType lookahead(int k) { + PsiBuilder.Marker tmp = mark(); + for (int i = 0; i < k; i++) advance(); + + JetToken tt = tt(); + tmp.rollbackTo(); + return tt; + } + + protected void consumeIf(JetToken token) { + if (at(token)) advance(); // token + } +} diff --git a/idea/src/org/jetbrains/jet/lang/parsing/JetExpressionParsing.java b/idea/src/org/jetbrains/jet/lang/parsing/JetExpressionParsing.java new file mode 100644 index 00000000000..4b2e695917b --- /dev/null +++ b/idea/src/org/jetbrains/jet/lang/parsing/JetExpressionParsing.java @@ -0,0 +1,39 @@ +package org.jetbrains.jet.lang.parsing; + +/** + * @author abreslav + */ +public class JetExpressionParsing extends AbstractJetParsing { + public JetExpressionParsing(SemanticWitespaceAwarePsiBuilder builder) { + super(builder); + } + + /* + * expression + * : "(" expression ")" // see tupleLiteral + * : literalConstant + * : functionLiteral + * : tupleLiteral + * : listLiteral + * : mapLiteral + * : range + * : "null" + * : "this" ("<" type ">")? + * : memberAccessExpression + * : expressionWithPrecedences + * : match + * : if + * : "typeof" + * : "new" constructorInvocation + * : objectLiteral + * : declaration + * : jump + * : loop + * // block is syntactically equivalent to a functionLiteral with no parameters + * ; + */ + public void parseExpression() { + advance(); // TODO + } + +} diff --git a/idea/src/org/jetbrains/jet/lang/parsing/JetParser.java b/idea/src/org/jetbrains/jet/lang/parsing/JetParser.java index 6b82a66ebec..edad43bdbae 100644 --- a/idea/src/org/jetbrains/jet/lang/parsing/JetParser.java +++ b/idea/src/org/jetbrains/jet/lang/parsing/JetParser.java @@ -13,7 +13,7 @@ import org.jetbrains.jet.JetNodeTypes; public class JetParser implements PsiParser { @NotNull public ASTNode parse(IElementType iElementType, PsiBuilder psiBuilder) { - new JetParsing(psiBuilder).parseFile(); + new JetParsing(new SemanticWitespaceAwarePsiBuilder(psiBuilder)).parseFile(); return psiBuilder.getTreeBuilt(); } } diff --git a/idea/src/org/jetbrains/jet/lang/parsing/JetParsing.java b/idea/src/org/jetbrains/jet/lang/parsing/JetParsing.java index 88f0c3ed81f..51854e7319a 100644 --- a/idea/src/org/jetbrains/jet/lang/parsing/JetParsing.java +++ b/idea/src/org/jetbrains/jet/lang/parsing/JetParsing.java @@ -4,7 +4,6 @@ package org.jetbrains.jet.lang.parsing; import com.intellij.lang.PsiBuilder; -import com.intellij.lang.WhitespaceSkippedCallback; import com.intellij.psi.tree.IElementType; import com.intellij.psi.tree.TokenSet; import org.jetbrains.jet.JetNodeType; @@ -17,7 +16,7 @@ import java.util.Map; import static org.jetbrains.jet.JetNodeTypes.*; import static org.jetbrains.jet.lexer.JetTokens.*; -public class JetParsing { +public class JetParsing extends AbstractJetParsing { private static final Map MODIFIER_KEYWORD_MAP = new HashMap(); static { for (IElementType softKeyword : MODIFIER_KEYWORDS.getTypes()) { @@ -34,24 +33,11 @@ public class JetParsing { private static final TokenSet NAMESPACE_NAME_RECOVERY_SET = TokenSet.create(DOT, EOL_OR_SEMICOLON); private static final TokenSet TYPE_REF_FIRST = TokenSet.create(LBRACKET, IDENTIFIER, LBRACE, LPAR); - private final WhitespaceSkippedCallback myWhitespaceSkippedCallback = new WhitespaceSkippedCallback() { - public void onSkip(IElementType type, int start, int end) { - CharSequence whitespace = JetParsing.this.myBuilder.getOriginalText(); - for (int i = start; i < end; i++) { - char c = whitespace.charAt(i); - if (c == '\n') { - myEOLInLastWhitespace = true; - break; - } - } - } - }; - private final PsiBuilder myBuilder; - private boolean myEOLInLastWhitespace; + private final JetExpressionParsing myExpressionParsing; - public JetParsing(PsiBuilder myBuilder) { - this.myBuilder = myBuilder; - myBuilder.setWhitespaceSkippedCallback(myWhitespaceSkippedCallback); + public JetParsing(SemanticWitespaceAwarePsiBuilder builder) { + super(builder); + this.myExpressionParsing = new JetExpressionParsing(builder); } /* @@ -350,7 +336,7 @@ public class JetParsing { type = NAMED_ARGUMENT; } if (at(OUT_KEYWORD) || at(REF_KEYWORD)) advance(); // REF or OUT - parseExpression(); + myExpressionParsing.parseExpression(); argument.done(type); } @@ -545,7 +531,7 @@ public class JetParsing { if (at(BY_KEYWORD)) { advance(); // BY_KEYWORD - parseExpression(); + myExpressionParsing.parseExpression(); delegator.done(DELEGATOR_BY); } else if (at(LPAR)) { @@ -625,39 +611,11 @@ public class JetParsing { if (at(EQ)) { advance(); // EQ - parseExpression(); + myExpressionParsing.parseExpression(); } return true; } - /* - * expression - * : "(" expression ")" // see tupleLiteral - * : literalConstant - * : functionLiteral - * : tupleLiteral - * : listLiteral - * : mapLiteral - * : range - * : "null" - * : "this" ("<" type ">")? - * : memberAccessExpression - * : expressionWithPrecedences - * : match - * : if - * : "typeof" - * : "new" constructorInvocation - * : objectLiteral - * : declaration - * : jump - * : loop - * // block is syntactically equivalent to a functionLiteral with no parameters - * ; - */ - private void parseExpression() { - advance(); // TODO - } - /* * typeParameters * : ("<" typeParameter{","} ">" @@ -892,93 +850,4 @@ public class JetParsing { return true; } -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - private boolean expect(JetToken expectation, String message) { - return expect(expectation, message, null); - } - - private PsiBuilder.Marker mark() { - return myBuilder.mark(); - } - - private void error(String message) { - myBuilder.error(message); - } - - private boolean expect(JetToken expectation, String message, TokenSet recoverySet) { - if (at(expectation)) { - advance(); // expectation - return true; - } - - errorWithRecovery(message, recoverySet); - - return false; - } - - private void errorWithRecovery(String message, TokenSet recoverySet) { - JetToken tt = tt(); - if (recoverySet == null || recoverySet.contains(tt) - || (recoverySet.contains(EOL_OR_SEMICOLON) - && (tt == SEMICOLON || myEOLInLastWhitespace))) { - error(message); - } - else { - errorAndAdvance(message); - } - } - - private void errorAndAdvance(String message) { - PsiBuilder.Marker err = mark(); - advance(); // erroneous token - err.error(message); - } - - private boolean eof() { - return myBuilder.eof(); - } - - private void advance() { - myEOLInLastWhitespace = false; - myBuilder.advanceLexer(); - } - - private JetToken tt() { - return (JetToken) myBuilder.getTokenType(); - } - - private boolean at(final IElementType expectation) { - JetToken token = tt(); - if (token == expectation) return true; - if (expectation == EOL_OR_SEMICOLON) { - if (token == SEMICOLON) return true; - if (myEOLInLastWhitespace) return true; - } - if (token == IDENTIFIER && expectation instanceof JetKeywordToken) { - JetKeywordToken expectedKeyword = (JetKeywordToken) expectation; - if (expectedKeyword.isSoft() && expectedKeyword.getValue().equals(myBuilder.getTokenText())) { - myBuilder.remapCurrentToken(expectation); - return true; - } - } - return false; - } - - private IElementType lookahead(int k) { - PsiBuilder.Marker tmp = mark(); - for (int i = 0; i < k; i++) advance(); - - JetToken tt = tt(); - tmp.rollbackTo(); - return tt; - } - - private void consumeIf(JetToken token) { - if (at(token)) advance(); // token - } - } \ No newline at end of file diff --git a/idea/src/org/jetbrains/jet/lang/parsing/PsiBuilderAdapter.java b/idea/src/org/jetbrains/jet/lang/parsing/PsiBuilderAdapter.java new file mode 100644 index 00000000000..2edba4a3cb8 --- /dev/null +++ b/idea/src/org/jetbrains/jet/lang/parsing/PsiBuilderAdapter.java @@ -0,0 +1,122 @@ +package org.jetbrains.jet.lang.parsing; + +import com.intellij.lang.*; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.util.Key; +import com.intellij.psi.tree.IElementType; +import com.intellij.psi.tree.TokenSet; +import com.intellij.util.diff.FlyweightCapableTreeStructure; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Copied from JavaParserUtil + * + * @author abreslav + */ +public class PsiBuilderAdapter implements PsiBuilder { + protected final PsiBuilder myDelegate; + + public PsiBuilderAdapter(final PsiBuilder delegate) { + myDelegate = delegate; + } + + public Project getProject() { + return myDelegate.getProject(); + } + + public CharSequence getOriginalText() { + return myDelegate.getOriginalText(); + } + + public void advanceLexer() { + myDelegate.advanceLexer(); + } + + @Nullable + public IElementType getTokenType() { + return myDelegate.getTokenType(); + } + + public void setTokenTypeRemapper(final ITokenTypeRemapper remapper) { + myDelegate.setTokenTypeRemapper(remapper); + } + + @Override + public void setWhitespaceSkippedCallback(WhitespaceSkippedCallback callback) { + myDelegate.setWhitespaceSkippedCallback(callback); + } + + @Override + public void remapCurrentToken(IElementType type) { + myDelegate.remapCurrentToken(type); + } + + @Override + public IElementType lookAhead(int steps) { + return myDelegate.lookAhead(steps); + } + + @Nullable + @NonNls + public String getTokenText() { + return myDelegate.getTokenText(); + } + + public int getCurrentOffset() { + return myDelegate.getCurrentOffset(); + } + + public Marker mark() { + return myDelegate.mark(); + } + + public void error(final String messageText) { + myDelegate.error(messageText); + } + + public boolean eof() { + return myDelegate.eof(); + } + + public ASTNode getTreeBuilt() { + return myDelegate.getTreeBuilt(); + } + + public FlyweightCapableTreeStructure getLightTree() { + return myDelegate.getLightTree(); + } + + public void setDebugMode(final boolean dbgMode) { + myDelegate.setDebugMode(dbgMode); + } + + public void enforceCommentTokens(final TokenSet tokens) { + myDelegate.enforceCommentTokens(tokens); + } + + @Nullable + public LighterASTNode getLatestDoneMarker() { + return myDelegate.getLatestDoneMarker(); + } + + @Nullable + public T getUserData(@NotNull final Key key) { + return myDelegate.getUserData(key); + } + + public void putUserData(@NotNull final Key key, @Nullable final T value) { + myDelegate.putUserData(key, value); + } + + @Override + public T getUserDataUnprotected(@NotNull final Key key) { + return myDelegate.getUserDataUnprotected(key); + } + + @Override + public void putUserDataUnprotected(@NotNull Key key, @Nullable T value) { + myDelegate.putUserDataUnprotected(key, value); + } +} diff --git a/idea/src/org/jetbrains/jet/lang/parsing/SemanticWitespaceAwarePsiBuilder.java b/idea/src/org/jetbrains/jet/lang/parsing/SemanticWitespaceAwarePsiBuilder.java new file mode 100644 index 00000000000..9c379a78dd7 --- /dev/null +++ b/idea/src/org/jetbrains/jet/lang/parsing/SemanticWitespaceAwarePsiBuilder.java @@ -0,0 +1,41 @@ +package org.jetbrains.jet.lang.parsing; + + +import com.intellij.lang.PsiBuilder; +import com.intellij.lang.WhitespaceSkippedCallback; +import com.intellij.psi.tree.IElementType; + +/** + * @author abreslav + */ +public class SemanticWitespaceAwarePsiBuilder extends PsiBuilderAdapter { + private final WhitespaceSkippedCallback myWhitespaceSkippedCallback = new WhitespaceSkippedCallback() { + public void onSkip(IElementType type, int start, int end) { + CharSequence whitespace = getOriginalText(); + for (int i = start; i < end; i++) { + char c = whitespace.charAt(i); + if (c == '\n') { + myEOLInLastWhitespace = true; + break; + } + } + } + }; + + private boolean myEOLInLastWhitespace; + + public SemanticWitespaceAwarePsiBuilder(final PsiBuilder delegate) { + super(delegate); + delegate.setWhitespaceSkippedCallback(myWhitespaceSkippedCallback); + } + + @Override + public void advanceLexer() { + myEOLInLastWhitespace = false; + super.advanceLexer(); + } + + public boolean eolInLastWhitespace() { + return myEOLInLastWhitespace; + } +}