KT-1385 support multi line strings with templates

compiler/frontend/src/org/jetbrains/jet/JetNodeTypes.java

@@ -87,7 +87,6 @@ public interface JetNodeTypes {
     JetNodeType BOOLEAN_CONSTANT   = new JetNodeType("BOOLEAN_CONSTANT", JetConstantExpression.class);
     JetNodeType FLOAT_CONSTANT     = new JetNodeType("FLOAT_CONSTANT", JetConstantExpression.class);
     JetNodeType CHARACTER_CONSTANT = new JetNodeType("CHARACTER_CONSTANT", JetConstantExpression.class);
-    JetNodeType RAW_STRING_CONSTANT = new JetNodeType("STRING_CONSTANT", JetConstantExpression.class);
     JetNodeType INTEGER_CONSTANT   = new JetNodeType("INTEGER_CONSTANT", JetConstantExpression.class);
 
     JetNodeType STRING_TEMPLATE    = new JetNodeType("STRING_TEMPLATE", JetStringTemplateExpression.class);

compiler/frontend/src/org/jetbrains/jet/lang/parsing/JetExpressionParsing.java

@@ -48,7 +48,7 @@ public class JetExpressionParsing extends AbstractJetParsing {
     }
 
     private static final TokenSet TYPE_ARGUMENT_LIST_STOPPERS = TokenSet.create(
-            INTEGER_LITERAL, FLOAT_LITERAL, CHARACTER_LITERAL, OPEN_QUOTE, RAW_STRING_LITERAL,
+            INTEGER_LITERAL, FLOAT_LITERAL, CHARACTER_LITERAL, OPEN_QUOTE,
             PACKAGE_KEYWORD, AS_KEYWORD, TYPE_KEYWORD, TRAIT_KEYWORD, CLASS_KEYWORD, THIS_KEYWORD, VAL_KEYWORD, VAR_KEYWORD,
             FUN_KEYWORD, FOR_KEYWORD, NULL_KEYWORD,
             TRUE_KEYWORD, FALSE_KEYWORD, IS_KEYWORD, THROW_KEYWORD, RETURN_KEYWORD, BREAK_KEYWORD,
@@ -70,7 +70,7 @@ public class JetExpressionParsing extends AbstractJetParsing {
 
             // literal constant
             TRUE_KEYWORD, FALSE_KEYWORD,
-            OPEN_QUOTE, RAW_STRING_LITERAL,
+            OPEN_QUOTE,
             INTEGER_LITERAL, CHARACTER_LITERAL, FLOAT_LITERAL,
             NULL_KEYWORD,
 
@@ -701,9 +701,6 @@ public class JetExpressionParsing extends AbstractJetParsing {
         if (at(TRUE_KEYWORD) || at(FALSE_KEYWORD)) {
             parseOneTokenExpression(BOOLEAN_CONSTANT);
         }
-        else if (at(RAW_STRING_LITERAL)) {
-            parseOneTokenExpression(RAW_STRING_CONSTANT);
-        }
         else if (at(INTEGER_LITERAL)) {
             parseOneTokenExpression(INTEGER_CONSTANT);
         }

compiler/frontend/src/org/jetbrains/jet/lang/types/expressions/BasicExpressionTypingVisitor.java

@@ -161,9 +161,6 @@ public class BasicExpressionTypingVisitor extends ExpressionTypingVisitor {
         else if (elementType == JetNodeTypes.CHARACTER_CONSTANT) {
             value = compileTimeConstantResolver.getCharValue(text, context.expectedType);
         }
-        else if (elementType == JetNodeTypes.RAW_STRING_CONSTANT) {
-            value = compileTimeConstantResolver.getRawStringValue(text, context.expectedType);
-        }
         else if (elementType == JetNodeTypes.NULL) {
             value = compileTimeConstantResolver.getNullValue(context.expectedType);
         }

compiler/frontend/src/org/jetbrains/jet/lang/types/expressions/ExpressionTypingUtils.java

@@ -111,9 +111,6 @@ public class ExpressionTypingUtils {
         else if (constantType == JetNodeTypes.CHARACTER_CONSTANT) {
             return semanticServices.getStandardLibrary().getCharType();
         }
-        else if (constantType == JetNodeTypes.RAW_STRING_CONSTANT) {
-            return semanticServices.getStandardLibrary().getStringType();
-        }
         else if (constantType == JetNodeTypes.NULL) {
             return JetStandardClasses.getNullableNothingType();
         }

compiler/frontend/src/org/jetbrains/jet/lexer/Jet.flex

@@ -45,7 +45,7 @@ import org.jetbrains.jet.lexer.JetTokens;
 %eof{  return;
 %eof}
 
-%xstate STRING SHORT_TEMPLATE_ENTRY
+%xstate STRING RAW_STRING SHORT_TEMPLATE_ENTRY
 %state LONG_TEMPLATE_ENTRY
 
 DIGIT=[0-9]
@@ -110,13 +110,19 @@ LONG_TEMPLATE_ENTRY_END=\}
 
 // String templates
 
-\"                                     { pushState(STRING); return JetTokens.OPEN_QUOTE; }
+{THREE_QUO}                      { pushState(RAW_STRING); return JetTokens.OPEN_QUOTE; }
+<RAW_STRING> \n                  { return JetTokens.REGULAR_STRING_PART; }
+<RAW_STRING> \"                  { return JetTokens.REGULAR_STRING_PART; }
+<RAW_STRING> \\                  { return JetTokens.REGULAR_STRING_PART; }
+<RAW_STRING> {THREE_QUO}         { popState(); return JetTokens.CLOSING_QUOTE; }
 
-<STRING> \"                            { popState(); return JetTokens.CLOSING_QUOTE; }
+\"                          { pushState(STRING); return JetTokens.OPEN_QUOTE; }
 <STRING> \n                 { popState(); yypushback(1); return JetTokens.DANGLING_NEWLINE; }
-<STRING> {REGULAR_STRING_PART}         { return JetTokens.REGULAR_STRING_PART; }
+<STRING> \"                 { popState(); return JetTokens.CLOSING_QUOTE; }
 <STRING> {ESCAPE_SEQUENCE}  { return JetTokens.ESCAPE_SEQUENCE; }
-<STRING> {SHORT_TEMPLATE_ENTRY}        {
+
+<STRING, RAW_STRING> {REGULAR_STRING_PART}         { return JetTokens.REGULAR_STRING_PART; }
+<STRING, RAW_STRING> {SHORT_TEMPLATE_ENTRY}        {
                                                         pushState(SHORT_TEMPLATE_ENTRY);
                                                         yypushback(yylength() - 1);
                                                         return JetTokens.SHORT_TEMPLATE_ENTRY_START;
@@ -127,8 +133,8 @@ LONG_TEMPLATE_ENTRY_END=\}
 <SHORT_TEMPLATE_ENTRY> "this"          { popState(); return JetTokens.THIS_KEYWORD; }
 <SHORT_TEMPLATE_ENTRY> {IDENTIFIER}    { popState(); return JetTokens.IDENTIFIER; }
 
-<STRING> {LONELY_DOLLAR}               { return JetTokens.REGULAR_STRING_PART; }
+<STRING, RAW_STRING> {LONELY_DOLLAR}               { return JetTokens.REGULAR_STRING_PART; }
-<STRING> {LONG_TEMPLATE_ENTRY_START}   { pushState(LONG_TEMPLATE_ENTRY); return JetTokens.LONG_TEMPLATE_ENTRY_START; }
+<STRING, RAW_STRING> {LONG_TEMPLATE_ENTRY_START}   { pushState(LONG_TEMPLATE_ENTRY); return JetTokens.LONG_TEMPLATE_ENTRY_START; }
 
 <LONG_TEMPLATE_ENTRY> "{"              { lBraceCount++; return JetTokens.LBRACE; }
 <LONG_TEMPLATE_ENTRY> "}"              {
@@ -158,9 +164,6 @@ LONG_TEMPLATE_ENTRY_END=\}
 {CHARACTER_LITERAL} { return JetTokens.CHARACTER_LITERAL; }
 //{STRING_LITERAL} { return JetTokens.STRING_LITERAL; }
 
-// TODO: Decide what to do with """ ... """"
-{RAW_STRING_LITERAL} { return JetTokens.RAW_STRING_LITERAL; }
-
 "continue"   { return JetTokens.CONTINUE_KEYWORD ;}
 "package"    { return JetTokens.PACKAGE_KEYWORD ;}
 "return"     { return JetTokens.RETURN_KEYWORD ;}

compiler/frontend/src/org/jetbrains/jet/lexer/JetTokens.java

@@ -45,8 +45,6 @@ public interface JetTokens {
     JetToken LONG_TEMPLATE_ENTRY_END = new JetToken("LONG_TEMPLATE_ENTRY_END");
     JetToken DANGLING_NEWLINE = new JetToken("DANGLING_NEWLINE");
 
-    JetToken RAW_STRING_LITERAL = new JetToken("RAW_STRING_LITERAL");
-
     JetKeywordToken PACKAGE_KEYWORD = JetKeywordToken.keyword("package");
     JetKeywordToken AS_KEYWORD               = JetKeywordToken.keyword("as");
     JetKeywordToken TYPE_KEYWORD             = JetKeywordToken.keyword("type");
@@ -185,7 +183,7 @@ public interface JetTokens {
     TokenSet WHITESPACES = TokenSet.create(TokenType.WHITE_SPACE);
     TokenSet COMMENTS = TokenSet.create(EOL_COMMENT, BLOCK_COMMENT, DOC_COMMENT);
 
-    TokenSet STRINGS = TokenSet.create(CHARACTER_LITERAL, REGULAR_STRING_PART, RAW_STRING_LITERAL);
+    TokenSet STRINGS = TokenSet.create(CHARACTER_LITERAL, REGULAR_STRING_PART);
     TokenSet OPERATIONS = TokenSet.create(AS_KEYWORD, AS_SAFE, IS_KEYWORD, IN_KEYWORD, DOT, PLUSPLUS, MINUSMINUS, MUL, PLUS,
             MINUS, EXCL, DIV, PERC, LT, GT, LTEQ, GTEQ, EQEQEQ, EXCLEQEQEQ, EQEQ, EXCLEQ, ANDAND, OROR,
             SAFE_ACCESS, ELVIS,

compiler/testData/codegen/multilineStringsWithTemplates.jet

@@ -0,0 +1,31 @@
+fun box() : String {
+    val s = "abc"
+    val test1 = """$s"""
+    if (test1 != "abc") return "Fail 1: $test1"
+
+    val test2 = """${s}"""
+    if (test2 != "abc") return "Fail 2: $test2"
+
+    val test3 = """ "$s" """
+    if (test3 != " \"abc\" ") return "Fail 3: $test3"
+
+    val test4 = """ "${s}" """
+    if (test4 != " \"abc\" ") return "Fail 4: $test4"
+
+    val test5 =
+"""
+  ${s.length}
+"""
+    if (test5 != "\n  3\n") return "Fail 5: $test5"
+
+    val test6 = """\n"""
+    if (test6 != "\\n") return "Fail 6: $test6"
+
+    val test7 = """\${'$'}foo"""
+    if (test7 != "\\\$foo") return "Fail 7: $test7"
+
+    val test8 = """$ foo"""
+    if (test8 != "$ foo") return "Fail 8: $test8"
+
+    return "OK"
+}

compiler/testData/diagnostics/tests/MultilineStringTemplates.jet

@@ -0,0 +1,43 @@
+fun box() : String {
+    val s = "abc"
+    val test1 = """$s"""
+    if (test1 != "abc") return "Fail 1: $test1"
+
+    val test2 = """${s}"""
+    if (test2 != "abc") return "Fail 2: $test2"
+
+    val test3 = """ "$s" """
+    if (test3 != " \"abc\" ") return "Fail 3: $test3"
+
+    val test4 = """ "${s}" """
+    if (test4 != " \"abc\" ") return "Fail 4: $test4"
+
+    val test5 =
+"""
+  ${s.length}
+"""
+    if (test5 != "\n  3\n") return "Fail 5: $test5"
+
+    val test6 = """\n"""
+    if (test6 != "\\n") return "Fail 6: $test6"
+
+    val test7 = """\${'$'}foo"""
+    if (test7 != "\\\$foo") return "Fail 7: $test7"
+
+    val test8 = """$ foo"""
+    if (test8 != "$ foo") return "Fail 8: $test8"
+
+    return "OK"
+}
+
+
+fun new() : String {
+  return """
+    sdfasdf
+    ${<!UNRESOLVED_REFERENCE!>a<!>}
+    ds"asdfas
+    $<!UNRESOLVED_REFERENCE!>b<!>
+    asgfaf
+  """
+
+}

compiler/testData/psi/SimpleExpressions.txt

@@ -105,8 +105,11 @@ JetFile: SimpleExpressions.jet
         PsiWhiteSpace(' ')
         PsiElement(EQ)('=')
         PsiWhiteSpace(' ')
-        STRING_CONSTANT
+        STRING_TEMPLATE
-          PsiElement(RAW_STRING_LITERAL)('"""dsf"""')
+          PsiElement(OPEN_QUOTE)('"""')
+          LITERAL_STRING_TEMPLATE_ENTRY
+            PsiElement(REGULAR_STRING_PART)('dsf')
+          PsiElement(CLOSING_QUOTE)('"""')
       PsiElement(COMMA)(',')
       PsiWhiteSpace('\n    ')
       VALUE_PARAMETER

compiler/testData/psi/When.txt

@@ -259,8 +259,11 @@ JetFile: When.jet
             PsiElement(is)('is')
             PsiWhiteSpace(' ')
             EXPRESSION_PATTERN
-              STRING_CONSTANT
+              STRING_TEMPLATE
-                PsiElement(RAW_STRING_LITERAL)('"""ddd"""')
+                PsiElement(OPEN_QUOTE)('"""')
+                LITERAL_STRING_TEMPLATE_ENTRY
+                  PsiElement(REGULAR_STRING_PART)('ddd')
+                PsiElement(CLOSING_QUOTE)('"""')
           PsiWhiteSpace(' ')
           PsiElement(ARROW)('->')
           PsiWhiteSpace(' ')

compiler/tests/org/jetbrains/jet/codegen/StringsTest.java

@@ -73,6 +73,10 @@ public class StringsTest extends CodegenTestCase {
         blackBoxFile("rawStrings.jet");
     }
 
+    public void testMultilineStringsWithTemplates() throws Exception {
+        blackBoxFile("multilineStringsWithTemplates.jet");
+    }
+
     public void testKt881() throws Exception {
         blackBoxFile("regressions/kt881.jet");
     }

idea/src/org/jetbrains/jet/plugin/JetHighlighter.java

@@ -192,7 +192,6 @@ public class JetHighlighter extends SyntaxHighlighterBase {
         keys1.put(JetTokens.ESCAPE_SEQUENCE, JET_STRING_ESCAPE);
 
         keys1.put(JetTokens.CHARACTER_LITERAL, JET_STRING);
-        keys1.put(JetTokens.RAW_STRING_LITERAL, JET_STRING);
 
         keys1.put(JetTokens.EOL_COMMENT, JET_COMMENT);
         keys1.put(JetTokens.BLOCK_COMMENT, JET_COMMENT);