From 3a2e30152a97ac03e1d4019f112189c978dc2cd6 Mon Sep 17 00:00:00 2001 From: Alexey Sedunov Date: Fri, 31 May 2013 19:32:21 +0400 Subject: [PATCH] Improve parser recovery for list of value parameters/arguments --- .../lang/parsing/JetExpressionParsing.java | 126 +++++++++--------- .../jet/lang/parsing/JetParsing.java | 7 +- .../org/jetbrains/jet/lexer/JetTokens.java | 40 +++--- .../MissingCommaInValueArgumentList.kt | 5 + .../MissingCommaInValueArgumentList.txt | 32 +++++ .../MissingCommaInValueParameterList.kt | 7 + .../MissingCommaInValueParameterList.txt | 44 ++++++ 7 files changed, 176 insertions(+), 85 deletions(-) create mode 100644 compiler/testData/psi/recovery/MissingCommaInValueArgumentList.kt create mode 100644 compiler/testData/psi/recovery/MissingCommaInValueArgumentList.txt create mode 100644 compiler/testData/psi/recovery/MissingCommaInValueParameterList.kt create mode 100644 compiler/testData/psi/recovery/MissingCommaInValueParameterList.txt diff --git a/compiler/frontend/src/org/jetbrains/jet/lang/parsing/JetExpressionParsing.java b/compiler/frontend/src/org/jetbrains/jet/lang/parsing/JetExpressionParsing.java index 1d2166f6e60..7fb709deed9 100644 --- a/compiler/frontend/src/org/jetbrains/jet/lang/parsing/JetExpressionParsing.java +++ b/compiler/frontend/src/org/jetbrains/jet/lang/parsing/JetExpressionParsing.java @@ -53,7 +53,7 @@ public class JetExpressionParsing extends AbstractJetParsing { TRUE_KEYWORD, FALSE_KEYWORD, IS_KEYWORD, THROW_KEYWORD, RETURN_KEYWORD, BREAK_KEYWORD, CONTINUE_KEYWORD, OBJECT_KEYWORD, IF_KEYWORD, TRY_KEYWORD, ELSE_KEYWORD, WHILE_KEYWORD, DO_KEYWORD, WHEN_KEYWORD, RBRACKET, RBRACE, RPAR, PLUSPLUS, MINUSMINUS, EXCLEXCL, -// MUL, + // MUL, PLUS, MINUS, EXCL, DIV, PERC, LTEQ, // TODO GTEQ, foo=x EQEQEQ, EXCLEQEQEQ, EQEQ, EXCLEQ, ANDAND, OROR, SAFE_ACCESS, ELVIS, @@ -111,17 +111,17 @@ public class JetExpressionParsing extends AbstractJetParsing { ); private static final TokenSet STATEMENT_FIRST = TokenSet.orSet( - EXPRESSION_FIRST, - TokenSet.create( - // declaration - LBRACKET, // attribute - FUN_KEYWORD, - VAL_KEYWORD, VAR_KEYWORD, - TRAIT_KEYWORD, - CLASS_KEYWORD, - TYPE_KEYWORD - ), - MODIFIER_KEYWORDS + EXPRESSION_FIRST, + TokenSet.create( + // declaration + LBRACKET, // attribute + FUN_KEYWORD, + VAL_KEYWORD, VAR_KEYWORD, + TRAIT_KEYWORD, + CLASS_KEYWORD, + TYPE_KEYWORD + ), + MODIFIER_KEYWORDS ); /*package*/ static final TokenSet EXPRESSION_FOLLOW = TokenSet.create( @@ -174,7 +174,7 @@ public class JetExpressionParsing extends AbstractJetParsing { EQUALITY(EQEQ, EXCLEQ, EQEQEQ, EXCLEQEQEQ), CONJUNCTION(ANDAND), DISJUNCTION(OROR), -// ARROW(JetTokens.ARROW), + // ARROW(JetTokens.ARROW), ASSIGNMENT(EQ, PLUSEQ, MINUSEQ, MULTEQ, DIVEQ, PERCEQ), ; @@ -287,7 +287,7 @@ public class JetExpressionParsing extends AbstractJetParsing { * see the precedence table */ private void parseBinaryExpression(Precedence precedence) { -// System.out.println(precedence.name() + " at " + myBuilder.getTokenText()); + // System.out.println(precedence.name() + " at " + myBuilder.getTokenText()); PsiBuilder.Marker expression = mark(); @@ -300,7 +300,7 @@ public class JetExpressionParsing extends AbstractJetParsing { JetNodeType resultType = precedence.parseRightHandSide(operation, this); expression.done(resultType); - expression = expression.precede(); + expression = expression.precede(); } expression.drop(); @@ -310,7 +310,7 @@ public class JetExpressionParsing extends AbstractJetParsing { * operation? prefixExpression */ private void parsePrefixExpression() { -// System.out.println("pre at " + myBuilder.getTokenText()); + // System.out.println("pre at " + myBuilder.getTokenText()); if (at(LBRACKET)) { if (!parseLocalDeclaration()) { @@ -485,10 +485,10 @@ public class JetExpressionParsing extends AbstractJetParsing { */ protected boolean parseCallWithClosure() { boolean success = false; -// while (!myBuilder.newlineBeforeCurrentToken() -// && (at(LBRACE) + // while (!myBuilder.newlineBeforeCurrentToken() + // && (at(LBRACE) while ((at(LBRACE) - || atSet(LABELS) && lookahead(1) == LBRACE)) { + || atSet(LABELS) && lookahead(1) == LBRACE)) { if (!at(LBRACE)) { assert _atSet(LABELS); parsePrefixExpression(); @@ -520,7 +520,7 @@ public class JetExpressionParsing extends AbstractJetParsing { * ; */ private void parseAtomicExpression() { -// System.out.println("atom at " + myBuilder.getTokenText()); + // System.out.println("atom at " + myBuilder.getTokenText()); if (at(LPAR)) { parseParenthesizedExpression(); @@ -574,7 +574,7 @@ public class JetExpressionParsing extends AbstractJetParsing { parseDoWhile(); } else if (atSet(CLASS_KEYWORD, FUN_KEYWORD, VAL_KEYWORD, - VAR_KEYWORD, TYPE_KEYWORD)) { + VAR_KEYWORD, TYPE_KEYWORD)) { parseLocalDeclaration(); } else if (at(FIELD_IDENTIFIER)) { @@ -784,7 +784,7 @@ public class JetExpressionParsing extends AbstractJetParsing { if (!at(ARROW)) { errorUntil("Expecting '->'", TokenSet.create(ARROW, - RBRACE, EOL_OR_SEMICOLON)); + RBRACE, EOL_OR_SEMICOLON)); } if (at(ARROW)) { @@ -798,7 +798,7 @@ public class JetExpressionParsing extends AbstractJetParsing { } } else if (!atSet(WHEN_CONDITION_RECOVERY_SET)) { - errorAndAdvance("Expecting '->'"); + errorAndAdvance("Expecting '->'"); } } else { @@ -982,8 +982,8 @@ public class JetExpressionParsing extends AbstractJetParsing { parseFunctionLiteralParametersAndType(); paramsFound = preferParamsToExpressions ? - rollbackOrDrop(rollbackMarker, ARROW, "An -> is expected", RBRACE) : - rollbackOrDropAt(rollbackMarker, ARROW); + rollbackOrDrop(rollbackMarker, ARROW, "An -> is expected", RBRACE) : + rollbackOrDropAt(rollbackMarker, ARROW); } if (!paramsFound) { @@ -1047,8 +1047,8 @@ public class JetExpressionParsing extends AbstractJetParsing { } private boolean rollbackOrDrop(PsiBuilder.Marker rollbackMarker, - JetToken expected, String expectMessage, - IElementType validForDrop) { + JetToken expected, String expectMessage, + IElementType validForDrop) { if (at(expected)) { advance(); // dropAt rollbackMarker.drop(); @@ -1074,8 +1074,8 @@ public class JetExpressionParsing extends AbstractJetParsing { while (!eof()) { PsiBuilder.Marker parameter = mark(); -// int parameterNamePos = matchTokenStreamPredicate(new LastBefore(new At(IDENTIFIER), new AtOffset(doubleArrowPos))); -// createTruncatedBuilder(parameterNamePos).parseModifierList(MODIFIER_LIST, false); + // int parameterNamePos = matchTokenStreamPredicate(new LastBefore(new At(IDENTIFIER), new AtOffset(doubleArrowPos))); + // createTruncatedBuilder(parameterNamePos).parseModifierList(MODIFIER_LIST, false); expect(IDENTIFIER, "Expecting parameter name", TokenSet.create(ARROW)); @@ -1135,8 +1135,8 @@ public class JetExpressionParsing extends AbstractJetParsing { } return preferParamsToExpressions ? - rollbackOrDrop(rollbackMarker, ARROW, "An -> is expected", RBRACE) : - rollbackOrDropAt(rollbackMarker, ARROW); + rollbackOrDrop(rollbackMarker, ARROW, "An -> is expected", RBRACE) : + rollbackOrDropAt(rollbackMarker, ARROW); } private void parseFunctionLiteralParametersAndType() { @@ -1252,38 +1252,38 @@ public class JetExpressionParsing extends AbstractJetParsing { * ; */ private IElementType parseLocalDeclarationRest(boolean isEnum) { - IElementType keywordToken = tt(); - IElementType declType = null; - if (keywordToken == CLASS_KEYWORD || keywordToken == TRAIT_KEYWORD) { - declType = myJetParsing.parseClass(isEnum); - } - else if (keywordToken == FUN_KEYWORD) { - declType = myJetParsing.parseFunction(); - } - else if (keywordToken == VAL_KEYWORD || keywordToken == VAR_KEYWORD) { - declType = myJetParsing.parseProperty(true); - } - else if (keywordToken == TYPE_KEYWORD) { - declType = myJetParsing.parseTypeDef(); - } - else if (keywordToken == OBJECT_KEYWORD) { - // Object expression may appear at the statement position: should parse it - // as expression instead of object declaration - // sample: - // { - // object : Thread() { - // } - // } - IElementType lookahead = lookahead(1); - if (lookahead == COLON || lookahead == LBRACE) { - return null; - } - - myJetParsing.parseObject(true, true); - declType = OBJECT_DECLARATION; - } - return declType; - } + IElementType keywordToken = tt(); + IElementType declType = null; + if (keywordToken == CLASS_KEYWORD || keywordToken == TRAIT_KEYWORD) { + declType = myJetParsing.parseClass(isEnum); + } + else if (keywordToken == FUN_KEYWORD) { + declType = myJetParsing.parseFunction(); + } + else if (keywordToken == VAL_KEYWORD || keywordToken == VAR_KEYWORD) { + declType = myJetParsing.parseProperty(true); + } + else if (keywordToken == TYPE_KEYWORD) { + declType = myJetParsing.parseTypeDef(); + } + else if (keywordToken == OBJECT_KEYWORD) { + // Object expression may appear at the statement position: should parse it + // as expression instead of object declaration + // sample: + // { + // object : Thread() { + // } + // } + IElementType lookahead = lookahead(1); + if (lookahead == COLON || lookahead == LBRACE) { + return null; + } + + myJetParsing.parseObject(true, true); + declType = OBJECT_DECLARATION; + } + return declType; + } /* * doWhile diff --git a/compiler/frontend/src/org/jetbrains/jet/lang/parsing/JetParsing.java b/compiler/frontend/src/org/jetbrains/jet/lang/parsing/JetParsing.java index dd519065f11..a56cd115ef9 100644 --- a/compiler/frontend/src/org/jetbrains/jet/lang/parsing/JetParsing.java +++ b/compiler/frontend/src/org/jetbrains/jet/lang/parsing/JetParsing.java @@ -50,6 +50,7 @@ public class JetParsing extends AbstractJetParsing { private static final TokenSet IMPORT_RECOVERY_SET = TokenSet.create(AS_KEYWORD, DOT, EOL_OR_SEMICOLON); /*package*/ static final TokenSet TYPE_REF_FIRST = TokenSet.create(LBRACKET, IDENTIFIER, FUN_KEYWORD, LPAR, CAPITALIZED_THIS_KEYWORD, HASH); private static final TokenSet RECEIVER_TYPE_TERMINATORS = TokenSet.create(DOT, SAFE_ACCESS); + private static final TokenSet VALUE_PARAMETER_FIRST = TokenSet.orSet(TokenSet.create(IDENTIFIER, LBRACKET), MODIFIER_KEYWORDS); static JetParsing createForTopLevel(SemanticWhitespaceAwarePsiBuilder builder) { JetParsing jetParsing = new JetParsing(builder); @@ -1722,8 +1723,10 @@ public class JetParsing extends AbstractJetParsing { else { parseValueParameter(); } - if (!at(COMMA)) break; - advance(); // COMMA + if (at(COMMA)) { + advance(); // COMMA + } + else if (!atSet(VALUE_PARAMETER_FIRST)) break; } } } diff --git a/compiler/frontend/src/org/jetbrains/jet/lexer/JetTokens.java b/compiler/frontend/src/org/jetbrains/jet/lexer/JetTokens.java index 6407280a4a1..0477cd27a1d 100644 --- a/compiler/frontend/src/org/jetbrains/jet/lexer/JetTokens.java +++ b/compiler/frontend/src/org/jetbrains/jet/lexer/JetTokens.java @@ -114,7 +114,7 @@ public interface JetTokens { JetToken SAFE_ACCESS = new JetToken("SAFE_ACCESS"); JetToken ELVIS = new JetToken("ELVIS"); // JetToken MAP = new JetToken("MAP"); -// JetToken FILTER = new JetToken("FILTER"); + // JetToken FILTER = new JetToken("FILTER"); JetToken QUEST = new JetToken("QUEST"); JetToken COLONCOLON = new JetToken("COLONCOLON"); JetToken COLON = new JetToken("COLON"); @@ -131,7 +131,7 @@ public interface JetTokens { JetToken HASH = new JetToken("HASH"); JetToken AT = new JetToken("AT"); JetToken ATAT = new JetToken("ATAT"); - + JetToken IDE_TEMPLATE_START = new JetToken("IDE_TEMPLATE_START"); JetToken IDE_TEMPLATE_END = new JetToken("IDE_TEMPLATE_END"); @@ -165,23 +165,23 @@ public interface JetTokens { JetKeywordToken FINAL_KEYWORD = JetKeywordToken.softKeyword("final"); TokenSet KEYWORDS = TokenSet.create(PACKAGE_KEYWORD, AS_KEYWORD, TYPE_KEYWORD, CLASS_KEYWORD, TRAIT_KEYWORD, - THIS_KEYWORD, SUPER_KEYWORD, VAL_KEYWORD, VAR_KEYWORD, FUN_KEYWORD, FOR_KEYWORD, - NULL_KEYWORD, - TRUE_KEYWORD, FALSE_KEYWORD, IS_KEYWORD, - IN_KEYWORD, THROW_KEYWORD, RETURN_KEYWORD, BREAK_KEYWORD, CONTINUE_KEYWORD, OBJECT_KEYWORD, IF_KEYWORD, - ELSE_KEYWORD, WHILE_KEYWORD, DO_KEYWORD, TRY_KEYWORD, WHEN_KEYWORD, - NOT_IN, NOT_IS, CAPITALIZED_THIS_KEYWORD, AS_SAFE + THIS_KEYWORD, SUPER_KEYWORD, VAL_KEYWORD, VAR_KEYWORD, FUN_KEYWORD, FOR_KEYWORD, + NULL_KEYWORD, + TRUE_KEYWORD, FALSE_KEYWORD, IS_KEYWORD, + IN_KEYWORD, THROW_KEYWORD, RETURN_KEYWORD, BREAK_KEYWORD, CONTINUE_KEYWORD, OBJECT_KEYWORD, IF_KEYWORD, + ELSE_KEYWORD, WHILE_KEYWORD, DO_KEYWORD, TRY_KEYWORD, WHEN_KEYWORD, + NOT_IN, NOT_IS, CAPITALIZED_THIS_KEYWORD, AS_SAFE ); TokenSet SOFT_KEYWORDS = TokenSet.create(IMPORT_KEYWORD, WHERE_KEYWORD, BY_KEYWORD, GET_KEYWORD, - SET_KEYWORD, ABSTRACT_KEYWORD, ENUM_KEYWORD, OPEN_KEYWORD, INNER_KEYWORD, ANNOTATION_KEYWORD, - OVERRIDE_KEYWORD, PRIVATE_KEYWORD, PUBLIC_KEYWORD, INTERNAL_KEYWORD, PROTECTED_KEYWORD, - CATCH_KEYWORD, FINALLY_KEYWORD, OUT_KEYWORD, FINAL_KEYWORD, VARARG_KEYWORD, INLINE_KEYWORD, REIFIED_KEYWORD + SET_KEYWORD, ABSTRACT_KEYWORD, ENUM_KEYWORD, OPEN_KEYWORD, INNER_KEYWORD, ANNOTATION_KEYWORD, + OVERRIDE_KEYWORD, PRIVATE_KEYWORD, PUBLIC_KEYWORD, INTERNAL_KEYWORD, PROTECTED_KEYWORD, + CATCH_KEYWORD, FINALLY_KEYWORD, OUT_KEYWORD, FINAL_KEYWORD, VARARG_KEYWORD, INLINE_KEYWORD, REIFIED_KEYWORD ); TokenSet MODIFIER_KEYWORDS = TokenSet.create(ABSTRACT_KEYWORD, ENUM_KEYWORD, - OPEN_KEYWORD, INNER_KEYWORD, ANNOTATION_KEYWORD, OVERRIDE_KEYWORD, PRIVATE_KEYWORD, PUBLIC_KEYWORD, INTERNAL_KEYWORD, - PROTECTED_KEYWORD, OUT_KEYWORD, IN_KEYWORD, FINAL_KEYWORD, VARARG_KEYWORD, INLINE_KEYWORD, REIFIED_KEYWORD + OPEN_KEYWORD, INNER_KEYWORD, ANNOTATION_KEYWORD, OVERRIDE_KEYWORD, PRIVATE_KEYWORD, PUBLIC_KEYWORD, INTERNAL_KEYWORD, + PROTECTED_KEYWORD, OUT_KEYWORD, IN_KEYWORD, FINAL_KEYWORD, VARARG_KEYWORD, INLINE_KEYWORD, REIFIED_KEYWORD ); TokenSet WHITESPACES = TokenSet.create(TokenType.WHITE_SPACE); @@ -196,13 +196,13 @@ public interface JetTokens { TokenSet STRINGS = TokenSet.create(CHARACTER_LITERAL, REGULAR_STRING_PART); TokenSet OPERATIONS = TokenSet.create(AS_KEYWORD, AS_SAFE, IS_KEYWORD, IN_KEYWORD, DOT, PLUSPLUS, MINUSMINUS, EXCLEXCL, MUL, PLUS, - MINUS, EXCL, DIV, PERC, LT, GT, LTEQ, GTEQ, EQEQEQ, EXCLEQEQEQ, EQEQ, EXCLEQ, ANDAND, OROR, - SAFE_ACCESS, ELVIS, -// MAP, FILTER, - COLON, - RANGE, EQ, MULTEQ, DIVEQ, PERCEQ, PLUSEQ, MINUSEQ, - NOT_IN, NOT_IS, - IDENTIFIER, LABEL_IDENTIFIER, ATAT, AT); + MINUS, EXCL, DIV, PERC, LT, GT, LTEQ, GTEQ, EQEQEQ, EXCLEQEQEQ, EQEQ, EXCLEQ, ANDAND, OROR, + SAFE_ACCESS, ELVIS, + // MAP, FILTER, + COLON, + RANGE, EQ, MULTEQ, DIVEQ, PERCEQ, PLUSEQ, MINUSEQ, + NOT_IN, NOT_IS, + IDENTIFIER, LABEL_IDENTIFIER, ATAT, AT); TokenSet AUGMENTED_ASSIGNMENTS = TokenSet.create(PLUSEQ, MINUSEQ, MULTEQ, PERCEQ, DIVEQ); TokenSet ALL_ASSIGNMENTS = TokenSet.create(EQ, PLUSEQ, MINUSEQ, MULTEQ, PERCEQ, DIVEQ); diff --git a/compiler/testData/psi/recovery/MissingCommaInValueArgumentList.kt b/compiler/testData/psi/recovery/MissingCommaInValueArgumentList.kt new file mode 100644 index 00000000000..38256a1b0e6 --- /dev/null +++ b/compiler/testData/psi/recovery/MissingCommaInValueArgumentList.kt @@ -0,0 +1,5 @@ +val x = foo( + a, + b + c +) \ No newline at end of file diff --git a/compiler/testData/psi/recovery/MissingCommaInValueArgumentList.txt b/compiler/testData/psi/recovery/MissingCommaInValueArgumentList.txt new file mode 100644 index 00000000000..76cd7f04939 --- /dev/null +++ b/compiler/testData/psi/recovery/MissingCommaInValueArgumentList.txt @@ -0,0 +1,32 @@ +JetFile: MissingCommaInValueArgumentList.kt + NAMESPACE_HEADER + + PROPERTY + PsiElement(val)('val') + PsiWhiteSpace(' ') + PsiElement(IDENTIFIER)('x') + PsiWhiteSpace(' ') + PsiElement(EQ)('=') + PsiWhiteSpace(' ') + CALL_EXPRESSION + REFERENCE_EXPRESSION + PsiElement(IDENTIFIER)('foo') + VALUE_ARGUMENT_LIST + PsiElement(LPAR)('(') + PsiWhiteSpace('\n ') + VALUE_ARGUMENT + REFERENCE_EXPRESSION + PsiElement(IDENTIFIER)('a') + PsiElement(COMMA)(',') + PsiWhiteSpace('\n ') + VALUE_ARGUMENT + BINARY_EXPRESSION + REFERENCE_EXPRESSION + PsiElement(IDENTIFIER)('b') + PsiWhiteSpace('\n ') + OPERATION_REFERENCE + PsiElement(IDENTIFIER)('c') + PsiErrorElement:Expecting an element + + PsiWhiteSpace('\n') + PsiElement(RPAR)(')') \ No newline at end of file diff --git a/compiler/testData/psi/recovery/MissingCommaInValueParameterList.kt b/compiler/testData/psi/recovery/MissingCommaInValueParameterList.kt new file mode 100644 index 00000000000..88e4a6d115a --- /dev/null +++ b/compiler/testData/psi/recovery/MissingCommaInValueParameterList.kt @@ -0,0 +1,7 @@ +fun foo( + a: Any, + b: Int + c: String +) { + +} \ No newline at end of file diff --git a/compiler/testData/psi/recovery/MissingCommaInValueParameterList.txt b/compiler/testData/psi/recovery/MissingCommaInValueParameterList.txt new file mode 100644 index 00000000000..6d69387549b --- /dev/null +++ b/compiler/testData/psi/recovery/MissingCommaInValueParameterList.txt @@ -0,0 +1,44 @@ +JetFile: MissingCommaInValueParameterList.kt + NAMESPACE_HEADER + + FUN + PsiElement(fun)('fun') + PsiWhiteSpace(' ') + PsiElement(IDENTIFIER)('foo') + VALUE_PARAMETER_LIST + PsiElement(LPAR)('(') + PsiWhiteSpace('\n ') + VALUE_PARAMETER + PsiElement(IDENTIFIER)('a') + PsiElement(COLON)(':') + PsiWhiteSpace(' ') + TYPE_REFERENCE + USER_TYPE + REFERENCE_EXPRESSION + PsiElement(IDENTIFIER)('Any') + PsiElement(COMMA)(',') + PsiWhiteSpace('\n ') + VALUE_PARAMETER + PsiElement(IDENTIFIER)('b') + PsiElement(COLON)(':') + PsiWhiteSpace(' ') + TYPE_REFERENCE + USER_TYPE + REFERENCE_EXPRESSION + PsiElement(IDENTIFIER)('Int') + PsiWhiteSpace('\n ') + VALUE_PARAMETER + PsiElement(IDENTIFIER)('c') + PsiElement(COLON)(':') + PsiWhiteSpace(' ') + TYPE_REFERENCE + USER_TYPE + REFERENCE_EXPRESSION + PsiElement(IDENTIFIER)('String') + PsiWhiteSpace('\n') + PsiElement(RPAR)(')') + PsiWhiteSpace(' ') + BLOCK + PsiElement(LBRACE)('{') + PsiWhiteSpace('\n\n') + PsiElement(RBRACE)('}') \ No newline at end of file