diff --git a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/parseCommandLineArguments.kt b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/parseCommandLineArguments.kt index 016ba600ce9..74a0c2ed9ae 100644 --- a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/parseCommandLineArguments.kt +++ b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/parseCommandLineArguments.kt @@ -52,7 +52,9 @@ data class ArgumentParseErrors( // Arguments where [Argument.deprecatedName] was used; the key is the deprecated name, the value is the new name ([Argument.value]) val deprecatedArguments: MutableMap = mutableMapOf(), - var argumentWithoutValue: String? = null + var argumentWithoutValue: String? = null, + + val argfileErrors: MutableList = SmartList() ) // Parses arguments into the passed [result] object. Errors related to the parsing will be collected into [CommonToolArguments.errors]. diff --git a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/preprocessCommandLineArguments.kt b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/preprocessCommandLineArguments.kt new file mode 100644 index 00000000000..ff2461517f2 --- /dev/null +++ b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/preprocessCommandLineArguments.kt @@ -0,0 +1,77 @@ +/* +* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license +* that can be found in the license/LICENSE.txt file. +*/ + +package org.jetbrains.kotlin.cli.common.arguments + +import java.io.* +import java.nio.charset.StandardCharsets + +private val experimentalArgfileArgument = "-Xargfile" +private val QUOTATION_MARK = '"' +private val BACKSLASH = '\\' +private val WHITESPACE = ' ' +private val NEWLINE = '\n' + +/** + * Performs initial preprocessing of arguments, passed to the compiler. + * This is done prior to *any* arguments parsing, and result of preprocessing + * will be used instead of actual passed arguments. + */ +fun preprocessCommandLineArguments(args: List, result: A): List = + args.flatMap { + if (it.isArgumentForArgfile) + File(it.argfilePath).expand(result) + else + listOf(it) + } + +private fun File.expand(result: A): List { + return try { + bufferedReader(Charsets.UTF_8).use { + generateSequence { it.parseNextArgument() }.toList() + } + } catch (e: FileNotFoundException) { + // Process FNFE separately to render absolutePath in error message + result.errors.argfileErrors += "Argfile not found: $absolutePath" + emptyList() + } catch (e: IOException) { + result.errors.argfileErrors += "Error while reading argfile: $e" + emptyList() + } +} + +private fun Reader.parseNextArgument(): String? { + val sb = StringBuilder() + + var r: Int = read() + while (r != -1) { + when (r.toChar()) { + WHITESPACE, NEWLINE -> return sb.toString() + QUOTATION_MARK -> consumeRestOfEscapedSequence(sb) + BACKSLASH -> sb.append(read().toChar()) + else -> sb.append(r.toChar()) + } + + r = read() + } + + return sb.toString().takeIf { it.isNotEmpty() } +} + +private fun Reader.consumeRestOfEscapedSequence(sb: StringBuilder) { + var ch = read().toChar() + while (ch != QUOTATION_MARK) { + if (ch == BACKSLASH) sb.append(read().toChar()) else sb.append(ch) + ch = read().toChar() + } +} + +private val String.argfilePath: String + get() = removePrefix("$experimentalArgfileArgument=") + +// Note that currently we use only experimental syntax for passing argfiles +// In 1.3 we can support also javac-like syntax `@argfile` +private val String.isArgumentForArgfile: Boolean + get() = startsWith("$experimentalArgfileArgument=") diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/common/CLITool.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/common/CLITool.kt index 9d74d8e2e5b..afcafe97e32 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/common/CLITool.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/common/CLITool.kt @@ -17,10 +17,7 @@ package org.jetbrains.kotlin.cli.common import org.fusesource.jansi.AnsiConsole -import org.jetbrains.kotlin.cli.common.arguments.ArgumentParseErrors -import org.jetbrains.kotlin.cli.common.arguments.CommonToolArguments -import org.jetbrains.kotlin.cli.common.arguments.parseCommandLineArguments -import org.jetbrains.kotlin.cli.common.arguments.validateArguments +import org.jetbrains.kotlin.cli.common.arguments.* import org.jetbrains.kotlin.cli.common.messages.* import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.INFO import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.STRONG_WARNING @@ -44,7 +41,8 @@ abstract class CLITool { args: Array ): ExitCode { val arguments = createArguments() - parseCommandLineArguments(args.asList(), arguments) + val preprocessedArguments = preprocessCommandLineArguments(args.asList(), arguments) + parseCommandLineArguments(preprocessedArguments, arguments) val collector = PrintingMessageCollector(errStream, messageRenderer, arguments.verbose) try { @@ -108,7 +106,8 @@ abstract class CLITool { // Used in kotlin-maven-plugin (KotlinCompileMojoBase) and in kotlin-gradle-plugin (KotlinJvmOptionsImpl, KotlinJsOptionsImpl) fun parseArguments(args: Array, arguments: A) { - parseCommandLineArguments(args.asList(), arguments) + val preprocessed = preprocessCommandLineArguments(args.asList(), arguments) + parseCommandLineArguments(preprocessed, arguments) val message = validateArguments(arguments.errors) if (message != null) { throw IllegalArgumentException(message) @@ -143,6 +142,9 @@ abstract class CLITool { "compiler or generated code. Use it at your own risk!\n" ) } + for (argfileError in errors.argfileErrors) { + collector.report(STRONG_WARNING, argfileError) + } } private fun printVersionIfNeeded(messageCollector: MessageCollector, arguments: A) { @@ -184,4 +186,4 @@ abstract class CLITool { } } } -} \ No newline at end of file +} diff --git a/compiler/preloader/src/org/jetbrains/kotlin/preloading/Preloader.java b/compiler/preloader/src/org/jetbrains/kotlin/preloading/Preloader.java index 4b3baf9f834..8cb1f2d39b7 100644 --- a/compiler/preloader/src/org/jetbrains/kotlin/preloading/Preloader.java +++ b/compiler/preloader/src/org/jetbrains/kotlin/preloading/Preloader.java @@ -266,10 +266,14 @@ public class Preloader { } } - private static class PreloaderException extends RuntimeException { + public static class PreloaderException extends RuntimeException { public PreloaderException(String message) { super(message); } + + public PreloaderException(String message, Throwable cause) { + super(message, cause); + } } private static class Handler extends ClassHandler { diff --git a/compiler/testData/cli/jvm/apiVersionLessThanLanguage.argfile b/compiler/testData/cli/jvm/apiVersionLessThanLanguage.argfile new file mode 100644 index 00000000000..4b325f26068 --- /dev/null +++ b/compiler/testData/cli/jvm/apiVersionLessThanLanguage.argfile @@ -0,0 +1 @@ +$TESTDATA_DIR$/apiVersion.kt -d $TEMP_DIR$ -api-version 1.0 -language-version 1.1 diff --git a/compiler/testData/cli/jvm/apiVersionLessThanLanguageUsingArgfile.args b/compiler/testData/cli/jvm/apiVersionLessThanLanguageUsingArgfile.args new file mode 100644 index 00000000000..ae9449ad7a5 --- /dev/null +++ b/compiler/testData/cli/jvm/apiVersionLessThanLanguageUsingArgfile.args @@ -0,0 +1 @@ +-Xargfile=$TESTDATA_DIR$/apiVersionLessThanLanguage.argfile \ No newline at end of file diff --git a/compiler/testData/cli/jvm/apiVersionLessThanLanguageUsingArgfile.out b/compiler/testData/cli/jvm/apiVersionLessThanLanguageUsingArgfile.out new file mode 100644 index 00000000000..47411cd0929 --- /dev/null +++ b/compiler/testData/cli/jvm/apiVersionLessThanLanguageUsingArgfile.out @@ -0,0 +1,7 @@ +compiler/testData/cli/jvm/apiVersion.kt:2:5: error: the feature "bound callable references" is only available since API version 1.1 + ""::class.isInstance(42) + ^ +compiler/testData/cli/jvm/apiVersion.kt:2:15: error: unresolved reference: isInstance + ""::class.isInstance(42) + ^ +COMPILATION_ERROR diff --git a/compiler/testData/cli/jvm/argfileWithEscaping.argfile b/compiler/testData/cli/jvm/argfileWithEscaping.argfile new file mode 100644 index 00000000000..bf23c67b7eb --- /dev/null +++ b/compiler/testData/cli/jvm/argfileWithEscaping.argfile @@ -0,0 +1,4 @@ +-X"some escaped \" sequence \\" +$TESTDATA_DIR$/apiVersion.kt +-d +$TEMP_DIR$ -api-version 1.0 -language-version 1.1 \ No newline at end of file diff --git a/compiler/testData/cli/jvm/argfileWithEscaping.args b/compiler/testData/cli/jvm/argfileWithEscaping.args new file mode 100644 index 00000000000..a3a729c19bd --- /dev/null +++ b/compiler/testData/cli/jvm/argfileWithEscaping.args @@ -0,0 +1 @@ +-Xargfile=$TESTDATA_DIR$/argfileWithEscaping.argfile \ No newline at end of file diff --git a/compiler/testData/cli/jvm/argfileWithEscaping.out b/compiler/testData/cli/jvm/argfileWithEscaping.out new file mode 100644 index 00000000000..45f6983f79d --- /dev/null +++ b/compiler/testData/cli/jvm/argfileWithEscaping.out @@ -0,0 +1,8 @@ +warning: flag is not supported by this version of the compiler: -Xsome escaped " sequence / +compiler/testData/cli/jvm/apiVersion.kt:2:5: error: the feature "bound callable references" is only available since API version 1.1 + ""::class.isInstance(42) + ^ +compiler/testData/cli/jvm/apiVersion.kt:2:15: error: unresolved reference: isInstance + ""::class.isInstance(42) + ^ +COMPILATION_ERROR diff --git a/compiler/testData/cli/jvm/importsProducerDump.txt b/compiler/testData/cli/jvm/importsProducerDump.txt index 5ebbf4c5c0c..41f02db5480 100644 --- a/compiler/testData/cli/jvm/importsProducerDump.txt +++ b/compiler/testData/cli/jvm/importsProducerDump.txt @@ -1 +1 @@ -{"\/home\/dsavvinov\/Repos\/kotlin-fork\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/a\/A1.kt":["import c.JavaC"],"\/home\/dsavvinov\/Repos\/kotlin-fork\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/a\/A2.kt":[],"\/home\/dsavvinov\/Repos\/kotlin-fork\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/b\/B1.kt":["import a.*"],"\/home\/dsavvinov\/Repos\/kotlin-fork\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/b\/nestedB\/B2.kt":["import a.A1 as AliasedA1","import a.A1"],"\/home\/dsavvinov\/Repos\/kotlin-fork\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/c\/C1.kt":["import a.A1","import a.A2","import b.B1.Companion.a2","import b.nestedB.bar","import b.nestedB.foo"]} \ No newline at end of file +{"\/home\/dsavvinov\/Repos\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/a\/A1.kt":["import c.JavaC"],"\/home\/dsavvinov\/Repos\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/a\/A2.kt":[],"\/home\/dsavvinov\/Repos\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/b\/B1.kt":["import a.*"],"\/home\/dsavvinov\/Repos\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/b\/nestedB\/B2.kt":["import a.A1 as AliasedA1","import a.A1"],"\/home\/dsavvinov\/Repos\/kotlin\/compiler\/testData\/cli\/jvm\/importsProducer\/c\/C1.kt":["import a.A1","import a.A2","import b.B1.Companion.a2","import b.nestedB.bar","import b.nestedB.foo"]} \ No newline at end of file diff --git a/compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.argfile b/compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.argfile new file mode 100644 index 00000000000..324174ac029 --- /dev/null +++ b/compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.argfile @@ -0,0 +1 @@ +$TEMP_DIR$ -api-version 1.0 -language-version 1.1 \ No newline at end of file diff --git a/compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.args b/compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.args new file mode 100644 index 00000000000..d69246c82dd --- /dev/null +++ b/compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.args @@ -0,0 +1,3 @@ +$TESTDATA_DIR$/apiVersion.kt +-d +-Xargfile=$TESTDATA_DIR$/mixingArgfilesAndUsualArgs.argfile \ No newline at end of file diff --git a/compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.out b/compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.out new file mode 100644 index 00000000000..47411cd0929 --- /dev/null +++ b/compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.out @@ -0,0 +1,7 @@ +compiler/testData/cli/jvm/apiVersion.kt:2:5: error: the feature "bound callable references" is only available since API version 1.1 + ""::class.isInstance(42) + ^ +compiler/testData/cli/jvm/apiVersion.kt:2:15: error: unresolved reference: isInstance + ""::class.isInstance(42) + ^ +COMPILATION_ERROR diff --git a/compiler/testData/cli/jvm/nonexistingArgfile.args b/compiler/testData/cli/jvm/nonexistingArgfile.args new file mode 100644 index 00000000000..8b0b8866e2d --- /dev/null +++ b/compiler/testData/cli/jvm/nonexistingArgfile.args @@ -0,0 +1,4 @@ +$TESTDATA_DIR$/simple.kt +-d +$TEMP_DIR$ +-Xargfile=$TESTDATA_DIR$/nonexisting.argfile \ No newline at end of file diff --git a/compiler/testData/cli/jvm/nonexistingArgfile.out b/compiler/testData/cli/jvm/nonexistingArgfile.out new file mode 100644 index 00000000000..1bea3649aff --- /dev/null +++ b/compiler/testData/cli/jvm/nonexistingArgfile.out @@ -0,0 +1,2 @@ +warning: argfile not found: $TESTDATA_DIR$/nonexisting.argfile +OK diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/cli/AbstractCliTest.java b/compiler/tests-common/tests/org/jetbrains/kotlin/cli/AbstractCliTest.java index fc61cfe856a..f9101a3097c 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/cli/AbstractCliTest.java +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/cli/AbstractCliTest.java @@ -17,6 +17,7 @@ package org.jetbrains.kotlin.cli; import com.intellij.openapi.util.io.FileUtil; +import com.intellij.openapi.util.io.FileUtilKt; import com.intellij.openapi.util.text.StringUtil; import kotlin.Pair; import kotlin.collections.CollectionsKt; @@ -30,6 +31,7 @@ import org.jetbrains.kotlin.cli.js.K2JSCompiler; import org.jetbrains.kotlin.cli.js.dce.K2JSDce; import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler; import org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler; +import org.jetbrains.kotlin.codegen.TestUtilsKt; import org.jetbrains.kotlin.config.KotlinCompilerVersion; import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion; import org.jetbrains.kotlin.test.CompilerTestUtil; @@ -42,8 +44,10 @@ import org.jetbrains.kotlin.utils.StringsKt; import org.junit.Assert; import java.io.File; +import java.nio.file.Files; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; public abstract class AbstractCliTest extends TestCaseWithTmpdir { private static final String TESTDATA_DIR = "$TESTDATA_DIR$"; @@ -180,28 +184,61 @@ public abstract class AbstractCliTest extends TestCaseWithTmpdir { } @NotNull - private static List readArgs(@NotNull String argsFilePath, @NotNull String tempDir) { - List lines = FilesKt.readLines(new File(argsFilePath), Charsets.UTF_8); + private static List readArgs(@NotNull String testArgsFilePath, @NotNull String tempDir) { + File testArgsFile = new File(testArgsFilePath); + List lines = FilesKt.readLines(testArgsFile, Charsets.UTF_8); + return CollectionsKt.mapNotNull(lines, arg -> readArg(arg, testArgsFile.getParent(), tempDir)); + } - return CollectionsKt.mapNotNull(lines, arg -> { - if (arg.isEmpty()) { - return null; - } + private static String readArg(String arg, @NotNull String testDataDir, @NotNull String tempDir) { + if (arg.isEmpty()) { + return null; + } - // Do not replace ':' after '\' (used in compiler plugin tests) - String argsWithColonsReplaced = arg - .replace("\\:", "$COLON$") - .replace(":", File.pathSeparator) - .replace("$COLON$", ":"); + String argWithColonsReplaced = arg + .replace("\\:", "$COLON$") + .replace(":", File.pathSeparator) + .replace("$COLON$", ":"); - return argsWithColonsReplaced - .replace("$TEMP_DIR$", tempDir) - .replace(TESTDATA_DIR, new File(argsFilePath).getParent()) - .replace( - "$FOREIGN_ANNOTATIONS_DIR$", - new File(AbstractForeignAnnotationsTestKt.getFOREIGN_ANNOTATIONS_SOURCES_PATH()).getPath() - ); - }); + String argWithTestPathsReplaced = replaceTestPaths(argWithColonsReplaced, testDataDir, tempDir); + + if (isArgfileArgument(arg)) { + return mockArgfile(argWithTestPathsReplaced, testDataDir, tempDir); + } else { + return argWithTestPathsReplaced; + } + } + + private static boolean isArgfileArgument(@NotNull String arg) { + return arg.startsWith("-Xargfile="); + } + + // Create new temp. argfile with all test paths replaced and return argfile-argument pointing to that file + private static String mockArgfile(@NotNull String argfileArgument, @NotNull String testDataDir, @NotNull String tempDir) { + int firstIndexOfArgfilePath = "-Xargfile=".length(); + String argfilePath = argfileArgument.substring(firstIndexOfArgfilePath); + File argfile = new File(argfilePath); + + if (argfile.exists()) { + File mockArgfile = FilesKt.createTempFile(argfile.getAbsolutePath(), "", new File(tempDir)); + String oldArgfileContent = FilesKt.readText(argfile, Charsets.UTF_8); + String newArgfileContent = replaceTestPaths(oldArgfileContent, testDataDir, tempDir); + FilesKt.writeText(mockArgfile, newArgfileContent, Charsets.UTF_8); + return "-Xargfile=" + mockArgfile.getAbsolutePath(); + } else { + return argfileArgument; + } + + } + + private static String replaceTestPaths(@NotNull String str, @NotNull String testDataDir, @NotNull String tempDir) { + return str + .replace("$TEMP_DIR$", tempDir) + .replace(TESTDATA_DIR, testDataDir) + .replace( + "$FOREIGN_ANNOTATIONS_DIR$", + new File(AbstractForeignAnnotationsTestKt.getFOREIGN_ANNOTATIONS_SOURCES_PATH()).getPath() + ); } protected void doJvmTest(@NotNull String fileName) { diff --git a/compiler/tests/org/jetbrains/kotlin/cli/CliTestGenerated.java b/compiler/tests/org/jetbrains/kotlin/cli/CliTestGenerated.java index 8e3a91b7889..2ea7fbdaa89 100644 --- a/compiler/tests/org/jetbrains/kotlin/cli/CliTestGenerated.java +++ b/compiler/tests/org/jetbrains/kotlin/cli/CliTestGenerated.java @@ -61,6 +61,16 @@ public class CliTestGenerated extends AbstractCliTest { runTest("compiler/testData/cli/jvm/apiVersionLessThanLanguage.args"); } + @TestMetadata("apiVersionLessThanLanguageUsingArgfile.args") + public void testApiVersionLessThanLanguageUsingArgfile() throws Exception { + runTest("compiler/testData/cli/jvm/apiVersionLessThanLanguageUsingArgfile.args"); + } + + @TestMetadata("argfileWithEscaping.args") + public void testArgfileWithEscaping() throws Exception { + runTest("compiler/testData/cli/jvm/argfileWithEscaping.args"); + } + @TestMetadata("argumentPassedMultipleTimes.args") public void testArgumentPassedMultipleTimes() throws Exception { runTest("compiler/testData/cli/jvm/argumentPassedMultipleTimes.args"); @@ -386,6 +396,11 @@ public class CliTestGenerated extends AbstractCliTest { runTest("compiler/testData/cli/jvm/legacySmartCastsAfterTry.args"); } + @TestMetadata("mixingArgfilesAndUsualArgs.args") + public void testMixingArgfilesAndUsualArgs() throws Exception { + runTest("compiler/testData/cli/jvm/mixingArgfilesAndUsualArgs.args"); + } + @TestMetadata("multipleTextRangesInDiagnosticsOrder.args") public void testMultipleTextRangesInDiagnosticsOrder() throws Exception { runTest("compiler/testData/cli/jvm/multipleTextRangesInDiagnosticsOrder.args"); @@ -426,6 +441,11 @@ public class CliTestGenerated extends AbstractCliTest { runTest("compiler/testData/cli/jvm/nonexistentScript.args"); } + @TestMetadata("nonexistingArgfile.args") + public void testNonexistingArgfile() throws Exception { + runTest("compiler/testData/cli/jvm/nonexistingArgfile.args"); + } + @TestMetadata("pluginSimple.args") public void testPluginSimple() throws Exception { runTest("compiler/testData/cli/jvm/pluginSimple.args");