Support argfiles in kotlin compiler

Using '-Xargfile=path/to/argfile' will substitute
that argument with the content of argfile.

See KT-24472

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/parseCommandLineArguments.kt

@@ -52,7 +52,9 @@ data class ArgumentParseErrors(
     // Arguments where [Argument.deprecatedName] was used; the key is the deprecated name, the value is the new name ([Argument.value])
     val deprecatedArguments: MutableMap<String, String> = mutableMapOf(),
 
-    var argumentWithoutValue: String? = null
+    var argumentWithoutValue: String? = null,
+
+    val argfileErrors: MutableList<String> = SmartList<String>()
 )
 
 // Parses arguments into the passed [result] object. Errors related to the parsing will be collected into [CommonToolArguments.errors].

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/preprocessCommandLineArguments.kt

@@ -0,0 +1,77 @@
+/*
+* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+* that can be found in the license/LICENSE.txt file.
+*/
+
+package org.jetbrains.kotlin.cli.common.arguments
+
+import java.io.*
+import java.nio.charset.StandardCharsets
+
+private val experimentalArgfileArgument = "-Xargfile"
+private val QUOTATION_MARK = '"'
+private val BACKSLASH = '\\'
+private val WHITESPACE = ' '
+private val NEWLINE = '\n'
+
+/**
+ * Performs initial preprocessing of arguments, passed to the compiler.
+ * This is done prior to *any* arguments parsing, and result of preprocessing
+ * will be used instead of actual passed arguments.
+ */
+fun <A : CommonToolArguments> preprocessCommandLineArguments(args: List<String>, result: A): List<String> =
+    args.flatMap {
+        if (it.isArgumentForArgfile)
+            File(it.argfilePath).expand(result)
+        else
+            listOf(it)
+    }
+
+private fun <A : CommonToolArguments> File.expand(result: A): List<String> {
+    return try {
+        bufferedReader(Charsets.UTF_8).use {
+            generateSequence { it.parseNextArgument() }.toList()
+        }
+    } catch (e: FileNotFoundException) {
+        // Process FNFE separately to render absolutePath in error message
+        result.errors.argfileErrors += "Argfile not found: $absolutePath"
+        emptyList()
+    } catch (e: IOException) {
+        result.errors.argfileErrors += "Error while reading argfile: $e"
+        emptyList()
+    }
+}
+
+private fun Reader.parseNextArgument(): String? {
+    val sb = StringBuilder()
+
+    var r: Int = read()
+    while (r != -1) {
+        when (r.toChar()) {
+            WHITESPACE, NEWLINE -> return sb.toString()
+            QUOTATION_MARK -> consumeRestOfEscapedSequence(sb)
+            BACKSLASH -> sb.append(read().toChar())
+            else -> sb.append(r.toChar())
+        }
+
+        r = read()
+    }
+
+    return sb.toString().takeIf { it.isNotEmpty() }
+}
+
+private fun Reader.consumeRestOfEscapedSequence(sb: StringBuilder) {
+    var ch = read().toChar()
+    while (ch != QUOTATION_MARK) {
+        if (ch == BACKSLASH) sb.append(read().toChar()) else sb.append(ch)
+        ch = read().toChar()
+    }
+}
+
+private val String.argfilePath: String
+    get() = removePrefix("$experimentalArgfileArgument=")
+
+// Note that currently we use only experimental syntax for passing argfiles
+// In 1.3 we can support also javac-like syntax `@argfile`
+private val String.isArgumentForArgfile: Boolean
+    get() = startsWith("$experimentalArgfileArgument=")

compiler/cli/src/org/jetbrains/kotlin/cli/common/CLITool.kt

@@ -17,10 +17,7 @@
 package org.jetbrains.kotlin.cli.common
 
 import org.fusesource.jansi.AnsiConsole
-import org.jetbrains.kotlin.cli.common.arguments.ArgumentParseErrors
+import org.jetbrains.kotlin.cli.common.arguments.*
-import org.jetbrains.kotlin.cli.common.arguments.CommonToolArguments
-import org.jetbrains.kotlin.cli.common.arguments.parseCommandLineArguments
-import org.jetbrains.kotlin.cli.common.arguments.validateArguments
 import org.jetbrains.kotlin.cli.common.messages.*
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.INFO
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.STRONG_WARNING
@@ -44,7 +41,8 @@ abstract class CLITool<A : CommonToolArguments> {
         args: Array<out String>
     ): ExitCode {
         val arguments = createArguments()
-        parseCommandLineArguments(args.asList(), arguments)
+        val preprocessedArguments = preprocessCommandLineArguments(args.asList(), arguments)
+        parseCommandLineArguments(preprocessedArguments, arguments)
         val collector = PrintingMessageCollector(errStream, messageRenderer, arguments.verbose)
 
         try {
@@ -108,7 +106,8 @@ abstract class CLITool<A : CommonToolArguments> {
 
     // Used in kotlin-maven-plugin (KotlinCompileMojoBase) and in kotlin-gradle-plugin (KotlinJvmOptionsImpl, KotlinJsOptionsImpl)
     fun parseArguments(args: Array<out String>, arguments: A) {
-        parseCommandLineArguments(args.asList(), arguments)
+        val preprocessed = preprocessCommandLineArguments(args.asList(), arguments)
+        parseCommandLineArguments(preprocessed, arguments)
         val message = validateArguments(arguments.errors)
         if (message != null) {
             throw IllegalArgumentException(message)
@@ -143,6 +142,9 @@ abstract class CLITool<A : CommonToolArguments> {
                         "compiler or generated code. Use it at your own risk!\n"
             )
         }
+        for (argfileError in errors.argfileErrors) {
+            collector.report(STRONG_WARNING, argfileError)
+        }
     }
 
     private fun <A : CommonToolArguments> printVersionIfNeeded(messageCollector: MessageCollector, arguments: A) {
@@ -184,4 +186,4 @@ abstract class CLITool<A : CommonToolArguments> {
             }
         }
     }
-}
+}

compiler/preloader/src/org/jetbrains/kotlin/preloading/Preloader.java

@@ -266,10 +266,14 @@ public class Preloader {
         }
     }
 
-    private static class PreloaderException extends RuntimeException {
+    public static class PreloaderException extends RuntimeException {
         public PreloaderException(String message) {
             super(message);
         }
+
+        public PreloaderException(String message, Throwable cause) {
+            super(message, cause);
+        }
     }
 
     private static class Handler extends ClassHandler {

compiler/testData/cli/jvm/apiVersionLessThanLanguage.argfile

@@ -0,0 +1 @@
+$TESTDATA_DIR$/apiVersion.kt -d $TEMP_DIR$ -api-version 1.0 -language-version 1.1

compiler/testData/cli/jvm/apiVersionLessThanLanguageUsingArgfile.args

@@ -0,0 +1 @@
+-Xargfile=$TESTDATA_DIR$/apiVersionLessThanLanguage.argfile

compiler/testData/cli/jvm/apiVersionLessThanLanguageUsingArgfile.out

@@ -0,0 +1,7 @@
+compiler/testData/cli/jvm/apiVersion.kt:2:5: error: the feature "bound callable references" is only available since API version 1.1
+    ""::class.isInstance(42)
+    ^
+compiler/testData/cli/jvm/apiVersion.kt:2:15: error: unresolved reference: isInstance
+    ""::class.isInstance(42)
+              ^
+COMPILATION_ERROR

compiler/testData/cli/jvm/argfileWithEscaping.argfile

@@ -0,0 +1,4 @@
+-X"some escaped \" sequence \\"
+$TESTDATA_DIR$/apiVersion.kt
+-d
+$TEMP_DIR$ -api-version 1.0 -language-version 1.1

compiler/testData/cli/jvm/argfileWithEscaping.args

@@ -0,0 +1 @@
+-Xargfile=$TESTDATA_DIR$/argfileWithEscaping.argfile

compiler/testData/cli/jvm/argfileWithEscaping.out

@@ -0,0 +1,8 @@
+warning: flag is not supported by this version of the compiler: -Xsome escaped " sequence /
+compiler/testData/cli/jvm/apiVersion.kt:2:5: error: the feature "bound callable references" is only available since API version 1.1
+    ""::class.isInstance(42)
+    ^
+compiler/testData/cli/jvm/apiVersion.kt:2:15: error: unresolved reference: isInstance
+    ""::class.isInstance(42)
+              ^
+COMPILATION_ERROR

compiler/testData/cli/jvm/importsProducerDump.txt

@@ -1 +1 @@
-{"\/home\/dsavvinov\/Repos\/kotlin-fork\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/a\/A1.kt":["import c.JavaC"],"\/home\/dsavvinov\/Repos\/kotlin-fork\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/a\/A2.kt":[],"\/home\/dsavvinov\/Repos\/kotlin-fork\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/b\/B1.kt":["import a.*"],"\/home\/dsavvinov\/Repos\/kotlin-fork\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/b\/nestedB\/B2.kt":["import a.A1 as AliasedA1","import a.A1"],"\/home\/dsavvinov\/Repos\/kotlin-fork\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/c\/C1.kt":["import a.A1","import a.A2","import b.B1.Companion.a2","import b.nestedB.bar","import b.nestedB.foo"]}
+{"\/home\/dsavvinov\/Repos\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/a\/A1.kt":["import c.JavaC"],"\/home\/dsavvinov\/Repos\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/a\/A2.kt":[],"\/home\/dsavvinov\/Repos\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/b\/B1.kt":["import a.*"],"\/home\/dsavvinov\/Repos\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/b\/nestedB\/B2.kt":["import a.A1 as AliasedA1","import a.A1"],"\/home\/dsavvinov\/Repos\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/c\/C1.kt":["import a.A1","import a.A2","import b.B1.Companion.a2","import b.nestedB.bar","import b.nestedB.foo"]}

compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.argfile

@@ -0,0 +1 @@
+$TEMP_DIR$ -api-version 1.0 -language-version 1.1

compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.args

@@ -0,0 +1,3 @@
+$TESTDATA_DIR$/apiVersion.kt
+-d
+-Xargfile=$TESTDATA_DIR$/mixingArgfilesAndUsualArgs.argfile

compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.out

@@ -0,0 +1,7 @@
+compiler/testData/cli/jvm/apiVersion.kt:2:5: error: the feature "bound callable references" is only available since API version 1.1
+    ""::class.isInstance(42)
+    ^
+compiler/testData/cli/jvm/apiVersion.kt:2:15: error: unresolved reference: isInstance
+    ""::class.isInstance(42)
+              ^
+COMPILATION_ERROR

compiler/testData/cli/jvm/nonexistingArgfile.args

@@ -0,0 +1,4 @@
+$TESTDATA_DIR$/simple.kt
+-d
+$TEMP_DIR$
+-Xargfile=$TESTDATA_DIR$/nonexisting.argfile

compiler/testData/cli/jvm/nonexistingArgfile.out

@@ -0,0 +1,2 @@
+warning: argfile not found: $TESTDATA_DIR$/nonexisting.argfile
+OK

compiler/tests-common/tests/org/jetbrains/kotlin/cli/AbstractCliTest.java

@@ -17,6 +17,7 @@
 package org.jetbrains.kotlin.cli;
 
 import com.intellij.openapi.util.io.FileUtil;
+import com.intellij.openapi.util.io.FileUtilKt;
 import com.intellij.openapi.util.text.StringUtil;
 import kotlin.Pair;
 import kotlin.collections.CollectionsKt;
@@ -30,6 +31,7 @@ import org.jetbrains.kotlin.cli.js.K2JSCompiler;
 import org.jetbrains.kotlin.cli.js.dce.K2JSDce;
 import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler;
 import org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler;
+import org.jetbrains.kotlin.codegen.TestUtilsKt;
 import org.jetbrains.kotlin.config.KotlinCompilerVersion;
 import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion;
 import org.jetbrains.kotlin.test.CompilerTestUtil;
@@ -42,8 +44,10 @@ import org.jetbrains.kotlin.utils.StringsKt;
 import org.junit.Assert;
 
 import java.io.File;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 public abstract class AbstractCliTest extends TestCaseWithTmpdir {
     private static final String TESTDATA_DIR = "$TESTDATA_DIR$";
@@ -180,28 +184,61 @@ public abstract class AbstractCliTest extends TestCaseWithTmpdir {
     }
 
     @NotNull
-    private static List<String> readArgs(@NotNull String argsFilePath, @NotNull String tempDir) {
+    private static List<String> readArgs(@NotNull String testArgsFilePath, @NotNull String tempDir) {
-        List<String> lines = FilesKt.readLines(new File(argsFilePath), Charsets.UTF_8);
+        File testArgsFile = new File(testArgsFilePath);
+        List<String> lines = FilesKt.readLines(testArgsFile, Charsets.UTF_8);
+        return CollectionsKt.mapNotNull(lines, arg -> readArg(arg, testArgsFile.getParent(), tempDir));
+    }
 
-        return CollectionsKt.mapNotNull(lines, arg -> {
+    private static String readArg(String arg, @NotNull String testDataDir, @NotNull String tempDir) {
-            if (arg.isEmpty()) {
+        if (arg.isEmpty()) {
-                return null;
+            return null;
-            }
+        }
 
-            // Do not replace ':' after '\' (used in compiler plugin tests)
+        String argWithColonsReplaced = arg
-            String argsWithColonsReplaced = arg
+                .replace("\\:", "$COLON$")
-                    .replace("\\:", "$COLON$")
+                .replace(":", File.pathSeparator)
-                    .replace(":", File.pathSeparator)
+                .replace("$COLON$", ":");
-                    .replace("$COLON$", ":");
 
-            return argsWithColonsReplaced
+        String argWithTestPathsReplaced = replaceTestPaths(argWithColonsReplaced, testDataDir, tempDir);
-                    .replace("$TEMP_DIR$", tempDir)
+
-                    .replace(TESTDATA_DIR, new File(argsFilePath).getParent())
+        if (isArgfileArgument(arg)) {
-                    .replace(
+            return mockArgfile(argWithTestPathsReplaced, testDataDir, tempDir);
-                            "$FOREIGN_ANNOTATIONS_DIR$",
+        } else {
-                            new File(AbstractForeignAnnotationsTestKt.getFOREIGN_ANNOTATIONS_SOURCES_PATH()).getPath()
+            return argWithTestPathsReplaced;
-                    );
+        }
-        });
+    }
+
+    private static boolean isArgfileArgument(@NotNull String arg) {
+        return arg.startsWith("-Xargfile=");
+    }
+
+    // Create new temp. argfile with all test paths replaced and return argfile-argument pointing to that file
+    private static String mockArgfile(@NotNull String argfileArgument, @NotNull String testDataDir, @NotNull String tempDir) {
+        int firstIndexOfArgfilePath = "-Xargfile=".length();
+        String argfilePath = argfileArgument.substring(firstIndexOfArgfilePath);
+        File argfile = new File(argfilePath);
+
+        if (argfile.exists()) {
+            File mockArgfile = FilesKt.createTempFile(argfile.getAbsolutePath(), "", new File(tempDir));
+            String oldArgfileContent = FilesKt.readText(argfile, Charsets.UTF_8);
+            String newArgfileContent = replaceTestPaths(oldArgfileContent, testDataDir, tempDir);
+            FilesKt.writeText(mockArgfile, newArgfileContent, Charsets.UTF_8);
+            return "-Xargfile=" + mockArgfile.getAbsolutePath();
+        } else {
+            return argfileArgument;
+        }
+
+    }
+
+    private static String replaceTestPaths(@NotNull String str, @NotNull String testDataDir, @NotNull String tempDir) {
+        return str
+                .replace("$TEMP_DIR$", tempDir)
+                .replace(TESTDATA_DIR, testDataDir)
+                .replace(
+                        "$FOREIGN_ANNOTATIONS_DIR$",
+                        new File(AbstractForeignAnnotationsTestKt.getFOREIGN_ANNOTATIONS_SOURCES_PATH()).getPath()
+                );
     }
 
     protected void doJvmTest(@NotNull String fileName) {

compiler/tests/org/jetbrains/kotlin/cli/CliTestGenerated.java

@@ -61,6 +61,16 @@ public class CliTestGenerated extends AbstractCliTest {
             runTest("compiler/testData/cli/jvm/apiVersionLessThanLanguage.args");
         }
 
+        @TestMetadata("apiVersionLessThanLanguageUsingArgfile.args")
+        public void testApiVersionLessThanLanguageUsingArgfile() throws Exception {
+            runTest("compiler/testData/cli/jvm/apiVersionLessThanLanguageUsingArgfile.args");
+        }
+
+        @TestMetadata("argfileWithEscaping.args")
+        public void testArgfileWithEscaping() throws Exception {
+            runTest("compiler/testData/cli/jvm/argfileWithEscaping.args");
+        }
+
         @TestMetadata("argumentPassedMultipleTimes.args")
         public void testArgumentPassedMultipleTimes() throws Exception {
             runTest("compiler/testData/cli/jvm/argumentPassedMultipleTimes.args");
@@ -386,6 +396,11 @@ public class CliTestGenerated extends AbstractCliTest {
             runTest("compiler/testData/cli/jvm/legacySmartCastsAfterTry.args");
         }
 
+        @TestMetadata("mixingArgfilesAndUsualArgs.args")
+        public void testMixingArgfilesAndUsualArgs() throws Exception {
+            runTest("compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.args");
+        }
+
         @TestMetadata("multipleTextRangesInDiagnosticsOrder.args")
         public void testMultipleTextRangesInDiagnosticsOrder() throws Exception {
             runTest("compiler/testData/cli/jvm/multipleTextRangesInDiagnosticsOrder.args");
@@ -426,6 +441,11 @@ public class CliTestGenerated extends AbstractCliTest {
             runTest("compiler/testData/cli/jvm/nonexistentScript.args");
         }
 
+        @TestMetadata("nonexistingArgfile.args")
+        public void testNonexistingArgfile() throws Exception {
+            runTest("compiler/testData/cli/jvm/nonexistingArgfile.args");
+        }
+
         @TestMetadata("pluginSimple.args")
         public void testPluginSimple() throws Exception {
             runTest("compiler/testData/cli/jvm/pluginSimple.args");