Introduce language feature: underscores in numeric literals

#KT-2964 Fixed

compiler/frontend/src/org/jetbrains/kotlin/diagnostics/Errors.java

@@ -723,6 +723,7 @@ public interface Errors {
     DiagnosticFactory2<KtConstantExpression, String, KotlinType> CONSTANT_EXPECTED_TYPE_MISMATCH = DiagnosticFactory2.create(ERROR);
     DiagnosticFactory0<KtConstantExpression> INCORRECT_CHARACTER_LITERAL = DiagnosticFactory0.create(ERROR);
     DiagnosticFactory0<KtConstantExpression> EMPTY_CHARACTER_LITERAL = DiagnosticFactory0.create(ERROR);
+    DiagnosticFactory0<KtConstantExpression> ILLEGAL_UNDERSCORE = DiagnosticFactory0.create(ERROR);
     DiagnosticFactory1<KtConstantExpression, KtConstantExpression> TOO_MANY_CHARACTERS_IN_CHARACTER_LITERAL = DiagnosticFactory1.create(ERROR);
     DiagnosticFactory1<KtElement, KtElement> ILLEGAL_ESCAPE = DiagnosticFactory1.create(ERROR, CUT_CHAR_QUOTES);
     DiagnosticFactory1<KtConstantExpression, KotlinType> NULL_FOR_NONNULL_TYPE = DiagnosticFactory1.create(ERROR);

compiler/frontend/src/org/jetbrains/kotlin/diagnostics/rendering/DefaultErrorMessages.java

@@ -454,6 +454,7 @@ public class DefaultErrorMessages {
         MAP.put(FLOAT_LITERAL_OUT_OF_RANGE, "The value is out of range");
         MAP.put(INCORRECT_CHARACTER_LITERAL, "Incorrect character literal");
         MAP.put(EMPTY_CHARACTER_LITERAL, "Empty character literal");
+        MAP.put(ILLEGAL_UNDERSCORE, "Illegal underscore");
         MAP.put(TOO_MANY_CHARACTERS_IN_CHARACTER_LITERAL, "Too many characters in a character literal ''{0}''", ELEMENT_TEXT);
         MAP.put(ILLEGAL_ESCAPE, "Illegal escape: ''{0}''", ELEMENT_TEXT);
         MAP.put(NULL_FOR_NONNULL_TYPE, "Null can not be a value of a non-null type {0}", RENDER_TYPE);

compiler/frontend/src/org/jetbrains/kotlin/resolve/constants/evaluate/ConstantExpressionEvaluator.kt

@@ -16,7 +16,10 @@
 
 package org.jetbrains.kotlin.resolve.constants.evaluate
 
+import com.intellij.psi.tree.IElementType
 import com.intellij.psi.util.PsiTreeUtil
+import com.intellij.util.text.LiteralFormatUtil
+import org.jetbrains.kotlin.KtNodeType
 import org.jetbrains.kotlin.KtNodeTypes
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.descriptors.*
@@ -283,8 +286,7 @@ private class ConstantExpressionEvaluatorVisitor(
         if (nodeElementType == KtNodeTypes.NULL) return factory.createNullValue().wrap()
 
         val result: Any? = when (nodeElementType) {
-                               KtNodeTypes.INTEGER_CONSTANT -> parseLong(text)
+                               KtNodeTypes.INTEGER_CONSTANT, KtNodeTypes.FLOAT_CONSTANT -> parseNumericLiteral(text, nodeElementType)
-                               KtNodeTypes.FLOAT_CONSTANT -> parseFloatingLiteral(text)
                                KtNodeTypes.BOOLEAN_CONSTANT -> parseBoolean(text)
                                KtNodeTypes.CHARACTER_CONSTANT -> CompileTimeConstantChecker.parseChar(expression)
                                else -> throw IllegalArgumentException("Unsupported constant: " + expression)
@@ -765,7 +767,16 @@ private class ConstantExpressionEvaluatorVisitor(
 
 private fun hasLongSuffix(text: String) = text.endsWith('l') || text.endsWith('L')
 
-fun parseLong(text: String): Long? {
+private fun parseNumericLiteral(text: String, type: IElementType): Any? {
+    val canonicalText = LiteralFormatUtil.removeUnderscores(text)
+    return when (type) {
+        KtNodeTypes.INTEGER_CONSTANT -> parseLong(canonicalText)
+        KtNodeTypes.FLOAT_CONSTANT -> parseFloatingLiteral(canonicalText)
+        else -> null
+    }
+}
+
+private fun parseLong(text: String): Long? {
     try {
         fun substringLongSuffix(s: String) = if (hasLongSuffix(text)) s.substring(0, s.length - 1) else s
         fun parseLong(text: String, radix: Int) = java.lang.Long.parseLong(substringLongSuffix(text), radix)

compiler/frontend/src/org/jetbrains/kotlin/types/expressions/BasicExpressionTypingVisitor.java

@@ -17,6 +17,7 @@
 package org.jetbrains.kotlin.types.expressions;
 
 import com.google.common.collect.Lists;
+import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.psi.PsiElement;
 import com.intellij.psi.tree.IElementType;
 import com.intellij.psi.tree.TokenSet;
@@ -27,6 +28,7 @@ import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.kotlin.KtNodeTypes;
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns;
+import org.jetbrains.kotlin.config.LanguageFeature;
 import org.jetbrains.kotlin.descriptors.*;
 import org.jetbrains.kotlin.diagnostics.Diagnostic;
 import org.jetbrains.kotlin.diagnostics.Errors;
@@ -71,9 +73,12 @@ import org.jetbrains.kotlin.types.expressions.unqualifiedSuper.UnqualifiedSuperK
 import org.jetbrains.kotlin.util.OperatorNameConventions;
 import org.jetbrains.kotlin.util.slicedMap.WritableSlice;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static org.jetbrains.kotlin.builtins.FunctionTypesKt.isExtensionFunctionType;
 import static org.jetbrains.kotlin.builtins.FunctionTypesKt.isFunctionType;
@@ -185,6 +190,10 @@ public class BasicExpressionTypingVisitor extends ExpressionTypingVisitor {
             checkLiteralPrefixAndSuffix(expression, context);
         }
 
+        if (elementType == KtNodeTypes.INTEGER_CONSTANT || elementType == KtNodeTypes.FLOAT_CONSTANT) {
+            checkUnderscores(expression, elementType, context);
+        }
+
         CompileTimeConstant<?> compileTimeConstant = components.constantExpressionEvaluator.evaluateExpression(
                 expression, context.trace, context.expectedType
         );
@@ -206,6 +215,49 @@ public class BasicExpressionTypingVisitor extends ExpressionTypingVisitor {
         return components.dataFlowAnalyzer.createCompileTimeConstantTypeInfo(compileTimeConstant, expression, context);
     }
 
+    private static final Pattern FP_LITERAL_PARTS = Pattern.compile("(?:([_\\d]*)\\.?([_\\d]*)e?[+-]?([_\\d]*))[f]?");
+
+    private void checkUnderscores(
+            @NotNull KtConstantExpression expression,
+            @NotNull IElementType elementType,
+            @NotNull ExpressionTypingContext context
+    ) {
+        String text = expression.getText().toLowerCase();
+
+        if (!text.contains("_")) return;
+
+        if (!components.languageVersionSettings.supportsFeature(LanguageFeature.UnderscoresInNumericLiterals)) {
+            context.trace.report(Errors.UNSUPPORTED_FEATURE.on(expression, LanguageFeature.UnderscoresInNumericLiterals));
+            return;
+        }
+
+        List<String> parts;
+
+        if (elementType == KtNodeTypes.INTEGER_CONSTANT) {
+            int start = 0;
+            int end = expression.getText().length();
+            if (text.startsWith("0x") || text.startsWith("0b")) start += 2;
+            if (StringUtil.endsWithChar(text, 'l')) --end;
+            parts = Collections.singletonList(text.substring(start, end));
+        }
+        else {
+            Matcher matcher = FP_LITERAL_PARTS.matcher(text);
+            parts = new ArrayList<String>();
+            if (matcher.matches()) {
+                for (int i = 0; i < matcher.groupCount(); i++) {
+                    parts.add(matcher.group(i + 1));
+                }
+            }
+        }
+
+        for (String part : parts) {
+            if (part != null && (part.startsWith("_") || part.endsWith("_"))) {
+                context.trace.report(Errors.ILLEGAL_UNDERSCORE.on(expression));
+                return;
+            }
+        }
+    }
+
     @NotNull
     public KotlinType getDefaultType(IElementType constantType) {
         KotlinBuiltIns builtIns = components.builtIns;

compiler/testData/diagnostics/tests/underscoresInNumericLiterals/illegalUnderscores.kt

@@ -0,0 +1,19 @@
+// !DIAGNOSTICS: -UNUSED_EXPRESSION
+
+fun foo() {
+    <!ILLEGAL_UNDERSCORE!>1_<!>
+    <!ILLEGAL_UNDERSCORE!>0x_f<!>
+    <!ILLEGAL_UNDERSCORE!>0X_f<!>
+    <!ILLEGAL_UNDERSCORE!>0b_1<!>
+    <!ILLEGAL_UNDERSCORE!>0B_1<!>
+    <!ILLEGAL_UNDERSCORE!>1.0_<!>
+    <!ILLEGAL_UNDERSCORE!>1_.1<!>
+    <!ILLEGAL_UNDERSCORE!>1._1<!>
+    <!ILLEGAL_UNDERSCORE!>1_._1<!>
+    <!ILLEGAL_UNDERSCORE!>1.0_e1<!>
+    <!ILLEGAL_UNDERSCORE!>1.0E_1<!>
+    <!ILLEGAL_UNDERSCORE!>0Xe_<!>
+    <!ILLEGAL_UNDERSCORE!>1.1_e-1_23<!>
+    <!ILLEGAL_UNDERSCORE!>1.0e+_0<!>
+    <!ILLEGAL_UNDERSCORE!>1.0e-_0<!>
+}

compiler/testData/diagnostics/tests/underscoresInNumericLiterals/illegalUnderscores.txt

@@ -0,0 +1,3 @@
+package
+
+public fun foo(): kotlin.Unit

compiler/testData/diagnostics/tests/underscoresInNumericLiterals/noUnderscores.kt

@@ -0,0 +1,10 @@
+// !LANGUAGE: -UnderscoresInNumericLiterals
+// !DIAGNOSTICS: -UNUSED_EXPRESSION
+
+fun foo() {
+    <!UNSUPPORTED_FEATURE!>100_1<!>
+    <!UNSUPPORTED_FEATURE!>3_.1<!>
+    <!UNSUPPORTED_FEATURE!>3_._1<!>
+    <!UNSUPPORTED_FEATURE!>2___4<!>
+    <!UNSUPPORTED_FEATURE!>123_<!>
+}

compiler/testData/diagnostics/tests/underscoresInNumericLiterals/noUnderscores.txt

@@ -0,0 +1,3 @@
+package
+
+public fun foo(): kotlin.Unit

compiler/tests/org/jetbrains/kotlin/checkers/DiagnosticsTestGenerated.java

@@ -21118,6 +21118,27 @@ public class DiagnosticsTestGenerated extends AbstractDiagnosticsTest {
             }
         }
 
+        @TestMetadata("compiler/testData/diagnostics/tests/underscoresInNumericLiterals")
+        @TestDataPath("$PROJECT_ROOT")
+        @RunWith(JUnit3RunnerWithInners.class)
+        public static class UnderscoresInNumericLiterals extends AbstractDiagnosticsTest {
+            public void testAllFilesPresentInUnderscoresInNumericLiterals() throws Exception {
+                KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("compiler/testData/diagnostics/tests/underscoresInNumericLiterals"), Pattern.compile("^(.+)\\.kt$"), true);
+            }
+
+            @TestMetadata("illegalUnderscores.kt")
+            public void testIllegalUnderscores() throws Exception {
+                String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/underscoresInNumericLiterals/illegalUnderscores.kt");
+                doTest(fileName);
+            }
+
+            @TestMetadata("noUnderscores.kt")
+            public void testNoUnderscores() throws Exception {
+                String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/underscoresInNumericLiterals/noUnderscores.kt");
+                doTest(fileName);
+            }
+        }
+
         @TestMetadata("compiler/testData/diagnostics/tests/unit")
         @TestDataPath("$PROJECT_ROOT")
         @RunWith(JUnit3RunnerWithInners.class)

compiler/util/src/org/jetbrains/kotlin/config/LanguageVersionSettings.kt

@@ -32,7 +32,8 @@ enum class LanguageFeature(val sinceVersion: LanguageVersion) {
     InlineProperties(KOTLIN_1_1),
     DestructuringLambdaParameters(KOTLIN_1_1),
     SingleUnderscoreForParameterName(KOTLIN_1_1),
-    DslMarkersSupport(KOTLIN_1_1)
+    DslMarkersSupport(KOTLIN_1_1),
+    UnderscoresInNumericLiterals(KOTLIN_1_1)
     ;
 
     val presentableText: String