Minor fixes. The semantic newline problem is still there

idea/src/org/jetbrains/jet/JetNodeTypes.java

@@ -64,6 +64,7 @@ public interface JetNodeTypes {
     JetNodeType SUPERTYE_QUALIFIER = new JetNodeType("SUPERTYE_QUALIFIER");
     JetNodeType LONG_CONSTANT = new JetNodeType("LONG_CONSTANT");
     JetNodeType TUPLE = new JetNodeType("TUPLE");
+    JetNodeType ARROW_TUPLE = new JetNodeType("ARROW_TUPLE");
     JetNodeType MAP_LITERAL_ENTRY = new JetNodeType("MAP_LITERAL_ENTRY");
     JetNodeType MAP_LITERAL = new JetNodeType("MAP_LITERAL");
     JetNodeType RANGE_LITERAL = new JetNodeType("RANGE_LITERAL");

idea/src/org/jetbrains/jet/lang/parsing/AbstractJetParsing.java

@@ -153,6 +153,7 @@ import static org.jetbrains.jet.lexer.JetTokens.*;
         if (at(token)) advance(); // token
     }
 
+    // TODO: Migrate to predicates
     protected void skipUntil(TokenSet tokenSet) {
         boolean stopAtEolOrSemi = tokenSet.contains(EOL_OR_SEMICOLON);
         while (!eof() && !tokenSet.contains(tt()) && !(stopAtEolOrSemi && at(EOL_OR_SEMICOLON))) {
@@ -183,10 +184,7 @@ import static org.jetbrains.jet.lexer.JetTokens.*;
         while (!eof()) {
             if (pattern.processToken(
                     myBuilder.getCurrentOffset(),
-                    openAngleBrackets == 0
+                    pattern.isTopLevel(openAngleBrackets, openBrackets, openBraces, openParentheses))) {
-                    && openBrackets == 0
-                    && openBraces == 0
-                    && openParentheses == 0)) {
                 break;
             }
             if (at(LPAR)) {
@@ -224,6 +222,8 @@ import static org.jetbrains.jet.lexer.JetTokens.*;
      * top-level occurrence of a token from the <code>stopSet</code>
      *
      * Returns -1 if no occurrence is found
+     *
+     * TODO: Migrate to predictaes
      */
     protected int findLastBefore(TokenSet lookFor, TokenSet stopAt, boolean dontStopRightAfterOccurrence) {
         return matchTokenStreamPredicate(new LastBefore(new AtSet(lookFor), new AtSet(stopAt), dontStopRightAfterOccurrence));

idea/src/org/jetbrains/jet/lang/parsing/AbstractTokenStreamPattern.java

@@ -5,11 +5,16 @@ package org.jetbrains.jet.lang.parsing;
  */
 public abstract class AbstractTokenStreamPattern implements TokenStreamPattern {
 
-        protected int lastOccurrence = -1;
+    protected int lastOccurrence = -1;
 
-        @Override
+    @Override
-        public int result() {
+    public int result() {
-            return lastOccurrence;
+        return lastOccurrence;
-        }
     }
 
+    @Override
+    public boolean isTopLevel(int openAngleBrackets, int openBrackets, int openBraces, int openParentheses) {
+        return openBraces == 0 && openBrackets == 0 && openParentheses == 0 && openAngleBrackets == 0;
+    }
+}
+

idea/src/org/jetbrains/jet/lang/parsing/JetExpressionParsing.java

@@ -21,10 +21,10 @@ public class JetExpressionParsing extends AbstractJetParsing {
     }
 
     @SuppressWarnings({"UnusedDeclaration"})
-    private enum Priority {
+    private enum Precedence {
         MEMBER_ACCESS(DOT, HASH, SAFE_ACCESS) {
             @Override
-            public void parseHigherPriority(JetExpressionParsing parsing) {
+            public void parseHigherPrecedence(JetExpressionParsing parsing) {
                 parsing.parseAtomicExpression();
             }
         },
@@ -34,14 +34,14 @@ public class JetExpressionParsing extends AbstractJetParsing {
         PREFIX(MINUS, PLUS, MINUSMINUS, PLUSPLUS, EXCL) { // attributes
 
             @Override
-            public void parseHigherPriority(JetExpressionParsing parsing) {
+            public void parseHigherPrecedence(JetExpressionParsing parsing) {
                 throw new IllegalStateException("Don't call this method");
             }
         },
 
         MULTIPLICATIVE(MUL, DIV, PERC) {
             @Override
-            public void parseHigherPriority(JetExpressionParsing parsing) {
+            public void parseHigherPrecedence(JetExpressionParsing parsing) {
                 parsing.parsePrefixExpression();
             }
         },
@@ -56,26 +56,27 @@ public class JetExpressionParsing extends AbstractJetParsing {
         CONJUNCTION(ANDAND),
         DISJUNCTION(OROR),
         MATCH(MATCH_KEYWORD),
+        // TODO: don't build a binary tree, build a tuple
+        ASSIGNMENT(EQ, PLUSEQ, MINUSEQ, MULTEQ, DIVEQ, PERCEQ),
         ARROW(JetTokens.ARROW),
-        ASSIGNMENT(EQ, PLUSEQ, MINUSEQ, MULTEQ, DIVEQ, PERCEQ)
         ;
 
         static {
-            Priority[] values = Priority.values();
+            Precedence[] values = Precedence.values();
-            for (Priority priority : values) {
+            for (Precedence precedence : values) {
-                int ordinal = priority.ordinal();
+                int ordinal = precedence.ordinal();
-                priority.higher = ordinal > 0 ? values[ordinal - 1] : null;
+                precedence.higher = ordinal > 0 ? values[ordinal - 1] : null;
             }
         }
 
-        private Priority higher;
+        private Precedence higher;
         private final TokenSet operations;
 
-        Priority(IElementType... operations ) {
+        Precedence(IElementType... operations) {
             this.operations = TokenSet.create(operations);
         }
 
-        public void parseHigherPriority(JetExpressionParsing parsing) {
+        public void parseHigherPrecedence(JetExpressionParsing parsing) {
             assert higher != null;
             parsing.parseBinaryExpression(higher);
         }
@@ -107,18 +108,23 @@ public class JetExpressionParsing extends AbstractJetParsing {
      *   ;
      */
     public void parseExpression() {
-        parseBinaryExpression(Priority.ASSIGNMENT);
+        parseBinaryExpression(Precedence.ASSIGNMENT);
     }
 
-    private void parseBinaryExpression(Priority priority) {
+    /*
-//        System.out.println(priority.name() + " at " + tt());
+     * expression (operation expression)*
+     *
+     * see the precedence table
+     */
+    private void parseBinaryExpression(Precedence precedence) {
+//        System.out.println(precedence.name() + " at " + tt());
         PsiBuilder.Marker expression = mark();
 
-        priority.parseHigherPriority(this);
+        precedence.parseHigherPrecedence(this);
 
-        while (!myBuilder.eolInLastWhitespace() && atSet(priority.getOperations())) {
+        while (!myBuilder.eolInLastWhitespace() && atSet(precedence.getOperations())) {
              advance(); // operation
-             priority.parseHigherPriority(this);
+             precedence.parseHigherPrecedence(this);
              expression.done(BINARY_EXPRESSION);
              expression = expression.precede();
         }
@@ -126,6 +132,9 @@ public class JetExpressionParsing extends AbstractJetParsing {
         expression.drop();
     }
 
+    /*
+     * operation? expression
+     */
     private void parsePrefixExpression() {
 //        System.out.println("pre at " + tt());
         if (at(LBRACKET)) {
@@ -135,7 +144,7 @@ public class JetExpressionParsing extends AbstractJetParsing {
                 parsePostfixExpression();
                 attributes.done(ANNOTATED_EXPRESSION);
             }
-        } else if (atSet(Priority.PREFIX.getOperations())) {
+        } else if (atSet(Precedence.PREFIX.getOperations())) {
             PsiBuilder.Marker expression = mark();
             advance(); // operation
             parsePostfixExpression();
@@ -145,17 +154,20 @@ public class JetExpressionParsing extends AbstractJetParsing {
         }
     }
 
+    /*
+     * expression operation?
+     */
     private void parsePostfixExpression() {
 //        System.out.println("post at " + tt());
 
         PsiBuilder.Marker expression = mark();
-        parseBinaryExpression(Priority.MEMBER_ACCESS);
+        parseBinaryExpression(Precedence.MEMBER_ACCESS);
         if (myBuilder.eolInLastWhitespace()) {
             expression.drop();
         } else if (at(LBRACKET)) {
             parseArrayAccess();
             expression.done(ARRAY_ACCESS_EXPRESSION);
-        } else if (atSet(Priority.POSTFIX.getOperations())) {
+        } else if (atSet(Precedence.POSTFIX.getOperations())) {
             advance(); // operation
             expression.done(POSTFIX_EXPRESSION);
         } else if (at(LPAR)) {
@@ -171,7 +183,7 @@ public class JetExpressionParsing extends AbstractJetParsing {
      *   : tupleLiteral // or parenthesized expression
      *   : "this" ("<" type ">")?
      *   : "typeof" "(" expression ")"
-     *   : "new" constructorInvocation // TODO: Do we need "new"?, see factory methods
+     *   : "new" constructorInvocation
      *   : objectLiteral
      *   : jump
      *   : if
@@ -347,7 +359,12 @@ public class JetExpressionParsing extends AbstractJetParsing {
 
         advance(); // LBRACE
 
-        int doubleArrowPos = matchTokenStreamPredicate(new FirstBefore(new At(DOUBLE_ARROW), new At(RBRACE)));
+        int doubleArrowPos = matchTokenStreamPredicate(new FirstBefore(new At(DOUBLE_ARROW), new At(RBRACE)) {
+            @Override
+            public boolean isTopLevel(int openAngleBrackets, int openBrackets, int openBraces, int openParentheses) {
+                return openBraces == 0;
+            }
+        });
 
         if (doubleArrowPos >= 0) {
             boolean dontExpectParameters = false;
@@ -460,7 +477,13 @@ public class JetExpressionParsing extends AbstractJetParsing {
     public void parseExpressions() {
         while (!eof() && !at(RBRACE)) {
             parseExpression();
-            consumeIf(SEMICOLON);
+            if (at(SEMICOLON)) {
+                advance(); // SEMICOLON
+            } else if (at(RBRACE)) {
+                break;
+            } else if (!myBuilder.eolInLastWhitespace()) {
+                errorUntil("Unexpected tokens (use ';' to separate expressions on the same line", TokenSet.create(EOL_OR_SEMICOLON));
+            }
         }
     }
 

idea/src/org/jetbrains/jet/lang/parsing/SemanticWhitespaceAwarePsiBuilderImpl.java

@@ -37,6 +37,6 @@ public class SemanticWhitespaceAwarePsiBuilderImpl extends PsiBuilderAdapter imp
 
     @Override
     public boolean eolInLastWhitespace() {
-        return myEOLInLastWhitespace;
+        return myEOLInLastWhitespace || eof();
     }
 }

idea/src/org/jetbrains/jet/lang/parsing/TokenStreamPattern.java

@@ -7,10 +7,8 @@ public interface TokenStreamPattern {
     /**
      * Called on each token
      *
-     *
-     *
      * @param offset
-     * @param topLevel indicates if no brackets (of types () [] {} <>) are currently unmatched
+     * @param topLevel see {@link #isTopLevel(int, int, int, int)}
      * @return <code>true</code> to stop
      */
     boolean processToken(int offset, boolean topLevel);
@@ -19,4 +17,10 @@ public interface TokenStreamPattern {
      * @return the position where the predicate has matched, -1 if no match was found
      */
     int result();
+
+    /*
+     * Decides if the combination of open bracet counts makes a "top level position"
+     * Straightforward meaning would be: if all counts are zero, then it's a top level
+     */
+    boolean isTopLevel(int openAngleBrackets, int openBrackets, int openBraces, int openParentheses);
 }

idea/testData/psi/FunctionLiterals_ERR.txt

@@ -42,16 +42,16 @@ JetFile: FunctionLiterals_ERR.jet
         PsiWhiteSpace('\n\n    ')
         FUNCTION_LITERAL
           PsiElement(LBRACE)('{')
-          BODY
+          VALUE_PARAMETER_LIST
-            TUPLE
+            PsiElement(LPAR)('(')
-              PsiElement(LPAR)('(')
+            VALUE_PARAMETER
               PsiElement(IDENTIFIER)('a')
-              PsiErrorElement:Expecting ')'
+            PsiErrorElement:Expecting ')
-                <empty list>
+              <empty list>
-            PsiWhiteSpace(' ')
+          PsiWhiteSpace(' ')
-            PsiErrorElement:Expecting an expression
+          PsiElement(DOUBLE_ARROW)('=>')
-              PsiElement(DOUBLE_ARROW)('=>')
+          PsiWhiteSpace(' ')
-            PsiWhiteSpace(' ')
+          BODY
             PsiElement(IDENTIFIER)('a')
           PsiElement(RBRACE)('}')
         PsiWhiteSpace('\n    ')