From 3f82192a7c86fc07f01f4c10c04835ad7cd00658 Mon Sep 17 00:00:00 2001 From: Andrey Breslav Date: Sun, 2 Jan 2011 17:00:52 +0300 Subject: [PATCH] EOLs are treated properly in the presence of comments and multiple whitespace tokens --- ...SemanticWhitespaceAwarePsiBuilderImpl.java | 22 +++-- idea/testData/psi/EOLsInComments.jet | 19 +++++ idea/testData/psi/EOLsInComments.txt | 84 +++++++++++++++++++ .../jetbrains/jet/parsing/JetParsingTest.java | 1 + 4 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 idea/testData/psi/EOLsInComments.jet create mode 100644 idea/testData/psi/EOLsInComments.txt diff --git a/idea/src/org/jetbrains/jet/lang/parsing/SemanticWhitespaceAwarePsiBuilderImpl.java b/idea/src/org/jetbrains/jet/lang/parsing/SemanticWhitespaceAwarePsiBuilderImpl.java index 0383064da58..07e465b23b0 100644 --- a/idea/src/org/jetbrains/jet/lang/parsing/SemanticWhitespaceAwarePsiBuilderImpl.java +++ b/idea/src/org/jetbrains/jet/lang/parsing/SemanticWhitespaceAwarePsiBuilderImpl.java @@ -4,6 +4,7 @@ package org.jetbrains.jet.lang.parsing; import com.intellij.lang.PsiBuilder; import com.intellij.psi.TokenType; import com.intellij.psi.tree.IElementType; +import org.jetbrains.jet.lexer.JetTokens; import java.util.Stack; @@ -22,14 +23,23 @@ public class SemanticWhitespaceAwarePsiBuilderImpl extends PsiBuilderAdapter imp public boolean eolInLastWhitespace() { if (!eolsEnabled.peek()) return false; if (eof()) return true; - IElementType previousToken = rawLookup(-1); - if (previousToken == TokenType.WHITE_SPACE) { - int previousTokenStart = rawTokenTypeStart(-1); - int previousTokenEnd = rawTokenTypeStart(0); + // TODO: maybe, memoize this somehow? + for (int i = 1; i <= getCurrentOffset(); i++) { + IElementType previousToken = rawLookup(-i); + if (previousToken == JetTokens.BLOCK_COMMENT + || previousToken == JetTokens.DOC_COMMENT + || previousToken == JetTokens.EOL_COMMENT) { + continue; + } + if (previousToken != TokenType.WHITE_SPACE) { + break; + } + int previousTokenStart = rawTokenTypeStart(-i); + int previousTokenEnd = rawTokenTypeStart(-i + 1); assert previousTokenStart >= 0; assert previousTokenEnd < getOriginalText().length(); - for (int i = previousTokenStart; i < previousTokenEnd; i++) { - if (getOriginalText().charAt(i) == '\n') return true; + for (int j = previousTokenStart; j < previousTokenEnd; j++) { + if (getOriginalText().charAt(j) == '\n') return true; } } return false; diff --git a/idea/testData/psi/EOLsInComments.jet b/idea/testData/psi/EOLsInComments.jet new file mode 100644 index 00000000000..8efb0ba238b --- /dev/null +++ b/idea/testData/psi/EOLsInComments.jet @@ -0,0 +1,19 @@ +fun foo() { + a + + b + a + /** */+ b + a + /* */+ b + a /* + */ + b + a + /* + */ + b + a /** + */ + b + a // + + b + a // ++ b +} \ No newline at end of file diff --git a/idea/testData/psi/EOLsInComments.txt b/idea/testData/psi/EOLsInComments.txt new file mode 100644 index 00000000000..7f8e9afbc9f --- /dev/null +++ b/idea/testData/psi/EOLsInComments.txt @@ -0,0 +1,84 @@ +JetFile: EOLsInComments.jet + NAMESPACE + FUN + PsiElement(fun)('fun') + PsiWhiteSpace(' ') + PsiElement(IDENTIFIER)('foo') + TYPE_PARAMETER_LIST + + VALUE_PARAMETER_LIST + PsiElement(LPAR)('(') + PsiElement(RPAR)(')') + PsiWhiteSpace(' ') + BLOCK + PsiElement(LBRACE)('{') + PsiWhiteSpace('\n ') + PsiElement(IDENTIFIER)('a') + PsiWhiteSpace('\n ') + PREFIX_EXPRESSION + PsiElement(PLUS)('+') + PsiWhiteSpace(' ') + PsiElement(IDENTIFIER)('b') + PsiWhiteSpace('\n ') + PsiElement(IDENTIFIER)('a') + PsiWhiteSpace('\n ') + PsiComment(DOC_COMMENT)('/** */') + PREFIX_EXPRESSION + PsiElement(PLUS)('+') + PsiWhiteSpace(' ') + PsiElement(IDENTIFIER)('b') + PsiWhiteSpace('\n ') + PsiElement(IDENTIFIER)('a') + PsiWhiteSpace('\n ') + PsiComment(BLOCK_COMMENT)('/* */') + PREFIX_EXPRESSION + PsiElement(PLUS)('+') + PsiWhiteSpace(' ') + PsiElement(IDENTIFIER)('b') + PsiWhiteSpace('\n ') + BINARY_EXPRESSION + PsiElement(IDENTIFIER)('a') + PsiWhiteSpace(' ') + PsiComment(BLOCK_COMMENT)('/*\n */') + PsiWhiteSpace(' ') + PsiElement(PLUS)('+') + PsiWhiteSpace(' ') + PsiElement(IDENTIFIER)('b') + PsiWhiteSpace('\n ') + PsiElement(IDENTIFIER)('a') + PsiWhiteSpace('\n ') + PsiComment(BLOCK_COMMENT)('/*\n */') + PsiWhiteSpace(' ') + PREFIX_EXPRESSION + PsiElement(PLUS)('+') + PsiWhiteSpace(' ') + PsiElement(IDENTIFIER)('b') + PsiWhiteSpace('\n ') + BINARY_EXPRESSION + PsiElement(IDENTIFIER)('a') + PsiWhiteSpace(' ') + PsiComment(DOC_COMMENT)('/**\n */') + PsiWhiteSpace(' ') + PsiElement(PLUS)('+') + PsiWhiteSpace(' ') + PsiElement(IDENTIFIER)('b') + PsiWhiteSpace('\n ') + PsiElement(IDENTIFIER)('a') + PsiWhiteSpace(' ') + PsiComment(EOL_COMMENT)('//') + PsiWhiteSpace('\n ') + PREFIX_EXPRESSION + PsiElement(PLUS)('+') + PsiWhiteSpace(' ') + PsiElement(IDENTIFIER)('b') + PsiWhiteSpace('\n ') + PsiElement(IDENTIFIER)('a') + PsiWhiteSpace(' ') + PsiComment(EOL_COMMENT)('//') + PsiWhiteSpace('\n') + PREFIX_EXPRESSION + PsiElement(PLUS)('+') + PsiWhiteSpace(' ') + PsiElement(IDENTIFIER)('b') + PsiWhiteSpace('\n') + PsiElement(RBRACE)('}') \ No newline at end of file diff --git a/idea/tests/org/jetbrains/jet/parsing/JetParsingTest.java b/idea/tests/org/jetbrains/jet/parsing/JetParsingTest.java index fa3c5b17a71..9ea79be05b4 100644 --- a/idea/tests/org/jetbrains/jet/parsing/JetParsingTest.java +++ b/idea/tests/org/jetbrains/jet/parsing/JetParsingTest.java @@ -38,6 +38,7 @@ public class JetParsingTest extends ParsingTestCase { public void testDecomposers_ERR() throws Exception {doTest(true);} public void testEmptyFile() throws Exception {doTest(true);} public void testEnums() throws Exception {doTest(true);} + public void testEOLsInComments() throws Exception {doTest(true);} public void testEOLsOnRollback() throws Exception {doTest(true);} public void testExtensions() throws Exception {doTest(true);} public void testExtensions_ERR() throws Exception {doTest(true);}