Introduce -Xcommon-sources and pass it correctly from build tool plugins

#KT-25196 In Progress
This commit is contained in:
Alexander Udalov
2018-07-27 14:15:12 +02:00
parent d35e73d29b
commit 0f003802fe
28 changed files with 205 additions and 126 deletions
@@ -180,6 +180,14 @@ abstract class CommonCompilerArguments : CommonToolArguments() {
)
var metadataVersion: String? by FreezableVar(null)
@Argument(
value = "-Xcommon-sources",
valueDescription = "<path>",
description = "Sources of the common module that need to be compiled together with this module in the multi-platform mode.\n" +
"Should be a subset of sources passed as free arguments"
)
var commonSources: Array<String>? by FreezableVar(null)
open fun configureAnalysisFlags(collector: MessageCollector): MutableMap<AnalysisFlag<*>, Any> {
return HashMap<AnalysisFlag<*>, Any>().apply {
put(AnalysisFlag.skipMetadataVersionCheck, skipMetadataVersionCheck)
@@ -21,11 +21,12 @@ import org.jetbrains.kotlin.modules.Module
import java.util.*
class ModuleBuilder(
private val name: String,
private val outputDir: String,
private val type: String
private val name: String,
private val outputDir: String,
private val type: String
) : Module {
private val sourceFiles = ArrayList<String>()
private val commonSourceFiles = ArrayList<String>()
private val classpathRoots = ArrayList<String>()
private val javaSourceRoots = ArrayList<JavaRootPath>()
private val friendDirs = ArrayList<String>()
@@ -35,6 +36,10 @@ class ModuleBuilder(
sourceFiles.add(path)
}
fun addCommonSourceFiles(path: String) {
commonSourceFiles.add(path)
}
fun addClasspathEntry(path: String) {
classpathRoots.add(path)
}
@@ -51,6 +56,7 @@ class ModuleBuilder(
override fun getFriendPaths(): List<String> = friendDirs
override fun getJavaSourceRoots(): List<JavaRootPath> = javaSourceRoots
override fun getSourceFiles(): List<String> = sourceFiles
override fun getCommonSourceFiles(): List<String> = commonSourceFiles
override fun getClasspathRoots(): List<String> = classpathRoots
override fun getModuleName(): String = name
override fun getModuleType(): String = type
@@ -47,6 +47,7 @@ public class ModuleXmlParser {
public static final String OUTPUT_DIR = "outputDir";
public static final String FRIEND_DIR = "friendDir";
public static final String SOURCES = "sources";
public static final String COMMON_SOURCES = "commonSources";
public static final String JAVA_SOURCE_ROOTS = "javaSourceRoots";
public static final String JAVA_SOURCE_PACKAGE_PREFIX = "packagePrefix";
public static final String PATH = "path";
@@ -159,6 +160,10 @@ public class ModuleXmlParser {
String path = getAttribute(attributes, PATH, qName);
moduleBuilder.addSourceFiles(path);
}
else if (COMMON_SOURCES.equalsIgnoreCase(qName)) {
String path = getAttribute(attributes, PATH, qName);
moduleBuilder.addCommonSourceFiles(path);
}
else if (FRIEND_DIR.equalsIgnoreCase(qName)) {
String path = getAttribute(attributes, PATH, qName);
moduleBuilder.addFriendDir(path);
@@ -10,14 +10,18 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
interface ContentRoot
data class KotlinSourceRoot(val path: String): ContentRoot
/**
* @param isCommon whether this source root contains sources of a common module in a multi-platform project
*/
data class KotlinSourceRoot(val path: String, val isCommon: Boolean): ContentRoot
fun CompilerConfiguration.addKotlinSourceRoot(source: String) {
add(CLIConfigurationKeys.CONTENT_ROOTS, KotlinSourceRoot(source))
@JvmOverloads
fun CompilerConfiguration.addKotlinSourceRoot(path: String, isCommon: Boolean = false) {
add(CLIConfigurationKeys.CONTENT_ROOTS, KotlinSourceRoot(path, isCommon))
}
fun CompilerConfiguration.addKotlinSourceRoots(sources: List<String>): Unit =
sources.forEach(this::addKotlinSourceRoot)
sources.forEach { addKotlinSourceRoot(it) }
val CompilerConfiguration.kotlinSourceRoots: List<String>
get() = get(CLIConfigurationKeys.CONTENT_ROOTS)?.filterIsInstance<KotlinSourceRoot>()?.map { it.path }.orEmpty()
val CompilerConfiguration.kotlinSourceRoots: List<KotlinSourceRoot>
get() = get(CLIConfigurationKeys.CONTENT_ROOTS)?.filterIsInstance<KotlinSourceRoot>().orEmpty()
@@ -26,6 +26,7 @@ import com.intellij.util.ExceptionUtil;
import com.intellij.util.SmartList;
import kotlin.collections.ArraysKt;
import kotlin.collections.CollectionsKt;
import kotlin.collections.SetsKt;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.kotlin.analyzer.AnalysisResult;
@@ -175,7 +176,12 @@ public class K2JSCompiler extends CLICompiler<K2JSCompilerArguments> {
configuration.put(JSConfigurationKeys.LIBRARIES, configureLibraries(arguments, paths, messageCollector));
ContentRootsKt.addKotlinSourceRoots(configuration, arguments.getFreeArgs());
String[] commonSourcesArray = arguments.getCommonSources();
Set<String> commonSources = commonSourcesArray == null ? Collections.emptySet() : SetsKt.setOf(commonSourcesArray);
for (String arg : arguments.getFreeArgs()) {
ContentRootsKt.addKotlinSourceRoot(configuration, arg, commonSources.contains(arg));
}
KotlinCoreEnvironment environmentForJS =
KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JS_CONFIG_FILES);
@@ -77,13 +77,14 @@ class K2JVMCompiler : CLICompiler<K2JVMCompilerArguments>() {
val pluginLoadResult = loadPlugins(arguments, configuration)
if (pluginLoadResult != ExitCode.OK) return pluginLoadResult
val commonSources = arguments.commonSources?.toSet().orEmpty()
if (!arguments.script && arguments.buildFile == null) {
for (arg in arguments.freeArgs) {
val file = File(arg)
if (file.extension == JavaFileType.DEFAULT_EXTENSION) {
configuration.addJavaSourceRoot(file)
} else {
configuration.addKotlinSourceRoot(arg)
configuration.addKotlinSourceRoot(arg, isCommon = arg in commonSources)
if (file.isDirectory) {
configuration.addJavaSourceRoot(file)
}
@@ -69,6 +69,7 @@ import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.common.CliModuleVisibilityManagerImpl
import org.jetbrains.kotlin.cli.common.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY
import org.jetbrains.kotlin.cli.common.config.ContentRoot
import org.jetbrains.kotlin.cli.common.config.KotlinSourceRoot
import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR
@@ -110,6 +111,7 @@ import org.jetbrains.kotlin.resolve.jvm.extensions.PackageFragmentProviderExtens
import org.jetbrains.kotlin.resolve.jvm.modules.JavaModuleResolver
import org.jetbrains.kotlin.resolve.lazy.declarations.CliDeclarationProviderFactoryService
import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactoryService
import org.jetbrains.kotlin.resolve.multiplatform.isCommonSource
import org.jetbrains.kotlin.script.ScriptDefinitionProvider
import org.jetbrains.kotlin.script.ScriptDependenciesProvider
import org.jetbrains.kotlin.script.ScriptReportSink
@@ -400,16 +402,18 @@ class KotlinCoreEnvironment private constructor(
private fun findJarRoot(file: File): VirtualFile? =
applicationEnvironment.jarFileSystem.findFileByPath("$file${URLUtil.JAR_SEPARATOR}")
private fun getSourceRootsCheckingForDuplicates(): Collection<String> {
val uniqueSourceRoots = linkedSetOf<String>()
private fun getSourceRootsCheckingForDuplicates(): List<KotlinSourceRoot> {
val uniqueSourceRoots = hashSetOf<String>()
val result = mutableListOf<KotlinSourceRoot>()
configuration.kotlinSourceRoots.forEach { path ->
if (!uniqueSourceRoots.add(path)) {
report(STRONG_WARNING, "Duplicate source root: $path")
for (root in configuration.kotlinSourceRoots) {
if (!uniqueSourceRoots.add(root.path)) {
report(STRONG_WARNING, "Duplicate source root: ${root.path}")
}
result.add(root)
}
return uniqueSourceRoots
return result
}
fun getSourceFiles(): List<KtFile> = sourceFiles
@@ -425,7 +429,7 @@ class KotlinCoreEnvironment private constructor(
val virtualFileCreator = PreprocessedFileCreator(project)
for (sourceRootPath in sourceRoots) {
for ((sourceRootPath, isCommon) in sourceRoots) {
val vFile = localFileSystem.findFileByPath(sourceRootPath)
if (vFile == null) {
val message = "Source file or directory not found: $sourceRootPath"
@@ -452,6 +456,9 @@ class KotlinCoreEnvironment private constructor(
val psiFile = psiManager.findFile(virtualFile)
if (psiFile is KtFile) {
result.add(psiFile)
if (isCommon) {
psiFile.isCommonSource = true
}
}
}
}
@@ -34,7 +34,7 @@ import org.jetbrains.kotlin.backend.jvm.JvmIrCodegenFactory
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.checkKotlinPackageUsage
import org.jetbrains.kotlin.cli.common.config.addKotlinSourceRoots
import org.jetbrains.kotlin.cli.common.config.addKotlinSourceRoot
import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.OUTPUT
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.WARNING
@@ -65,18 +65,6 @@ import java.lang.reflect.InvocationTargetException
import java.net.URLClassLoader
object KotlinToJVMBytecodeCompiler {
private fun getAbsoluteFiles(buildFile: File, module: Module): List<File> {
return module.getSourceFiles().map { sourceFile ->
val source = File(sourceFile)
if (!source.isAbsolute) {
File(buildFile.absoluteFile.parentFile, sourceFile)
} else {
source
}
}
}
private fun writeOutput(
configuration: CompilerConfiguration,
outputFiles: OutputFileCollection,
@@ -142,7 +130,7 @@ object KotlinToJVMBytecodeCompiler {
for (module in chunk) {
ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
val moduleSourceFiles = getAbsoluteFiles(buildFile, module).map { file -> localFileSystem.findFileByPath(file.path) }
val moduleSourceFiles = getAbsolutePaths(buildFile, module.getSourceFiles()).map(localFileSystem::findFileByPath)
val ktFiles = environment.getSourceFiles().filter { file -> file.virtualFile in moduleSourceFiles }
if (!checkKotlinPackageUsage(environment, ktFiles)) return false
@@ -184,7 +172,11 @@ object KotlinToJVMBytecodeCompiler {
internal fun configureSourceRoots(configuration: CompilerConfiguration, chunk: List<Module>, buildFile: File) {
for (module in chunk) {
configuration.addKotlinSourceRoots(getAbsoluteFiles(buildFile, module).map(File::getPath))
val commonSources = getAbsolutePaths(buildFile, module.getCommonSourceFiles()).toSet()
for (path in getAbsolutePaths(buildFile, module.getSourceFiles())) {
configuration.addKotlinSourceRoot(path, isCommon = path in commonSources)
}
}
for (module in chunk) {
@@ -224,6 +216,16 @@ object KotlinToJVMBytecodeCompiler {
configuration.addAll(JVMConfigurationKeys.MODULES, chunk)
}
private fun getAbsolutePaths(buildFile: File, sourceFilePaths: List<String>): List<String> =
sourceFilePaths.map { path ->
val sourceFile = File(path)
if (!sourceFile.isAbsolute) {
File(buildFile.absoluteFile.parentFile, path).absolutePath
} else {
sourceFile.absolutePath
}
}
private fun findMainClass(generationState: GenerationState, files: List<KtFile>): FqName? {
val mainFunctionDetector = MainFunctionDetector(generationState.bindingContext)
return files.asSequence()
@@ -60,7 +60,7 @@ class K2MetadataCompiler : CLICompiler<K2MetadataCompilerArguments>() {
if (pluginLoadResult != ExitCode.OK) return pluginLoadResult
for (arg in arguments.freeArgs) {
configuration.addKotlinSourceRoot(arg)
configuration.addKotlinSourceRoot(arg, isCommon = true)
}
if (arguments.classpath != null) {
configuration.addJvmClasspathRoots(arguments.classpath!!.split(File.pathSeparatorChar).map(::File))
@@ -38,7 +38,6 @@ import org.jetbrains.kotlin.cli.js.K2JSCompiler
import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler
import org.jetbrains.kotlin.config.IncrementalCompilation
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.daemon.report.CompileServicesFacadeMessageCollector
@@ -50,10 +49,10 @@ import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer
import org.jetbrains.kotlin.incremental.parsing.classesFqNames
import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryAndroid
import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryJs
import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryJvm
import org.jetbrains.kotlin.incremental.parsing.classesFqNames
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
import org.jetbrains.kotlin.modules.Module
import org.jetbrains.kotlin.progress.CompilationCanceledStatus
@@ -516,6 +515,8 @@ class CompileServiceImpl(
it.getJavaSourceRoots().map { JvmSourceRoot(File(it.path), it.packagePrefix) }
}
k2jvmArgs.commonSources = parsedModule.modules.flatMap { it.getCommonSourceFiles() }.toTypedArray().takeUnless { it.isEmpty() }
val allKotlinFiles = parsedModule.modules.flatMap { it.getSourceFiles().map(::File) }
k2jvmArgs.friendPaths = parsedModule.modules.flatMap(Module::getFriendPaths).toTypedArray()
@@ -0,0 +1,16 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.resolve.multiplatform
import com.intellij.openapi.util.Key
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.psi.UserDataProperty
/**
* Returns true if this file is a part of the common module in a multi-platform project.
* This setting only makes sense in the compiler, not in the IDE where sources from common modules are analyzed as common
*/
var KtFile.isCommonSource: Boolean? by UserDataProperty(Key.create("IS_COMMON_SOURCE"))
@@ -340,13 +340,16 @@ class IncrementalJvmCompilerRunner(
val compiler = K2JVMCompiler()
val outputDir = args.destinationAsFile
val classpath = args.classpathAsList
val moduleFile = makeModuleFile(args.moduleName!!,
isTest = false,
outputDir = outputDir,
sourcesToCompile = sourcesToCompile,
javaSourceRoots = javaSourceRoots,
classpath = classpath,
friendDirs = listOf())
val moduleFile = makeModuleFile(
args.moduleName!!,
isTest = false,
outputDir = outputDir,
sourcesToCompile = sourcesToCompile,
commonSources = args.commonSources?.map(::File).orEmpty(),
javaSourceRoots = javaSourceRoots,
classpath = classpath,
friendDirs = listOf()
)
val destination = args.destination
args.destination = null
args.buildFile = moduleFile.absolutePath
+2
View File
@@ -4,6 +4,8 @@ where advanced options include:
-Xfriend-modules-disabled Disable internal declaration export
-Xtyped-arrays Translate primitive arrays to JS typed arrays
-Xallow-kotlin-package Allow compiling code in package 'kotlin' and allow not requiring kotlin.stdlib in module-info
-Xcommon-sources=<path> Sources of the common module that need to be compiled together with this module in the multi-platform mode.
Should be a subset of sources passed as free arguments
-Xcoroutines={enable|warn|error}
Enable coroutines or report warnings or errors on declarations and use sites of 'suspend' modifier
-Xdump-perf=<path> Dump detailed performance statistics to the specified file
+2
View File
@@ -60,6 +60,8 @@ where advanced options include:
-Xuse-old-class-files-reading Use old class files reading implementation (may slow down the build and should be used in case of problems with the new implementation)
-Xuse-type-table Use type table in metadata serialization
-Xallow-kotlin-package Allow compiling code in package 'kotlin' and allow not requiring kotlin.stdlib in module-info
-Xcommon-sources=<path> Sources of the common module that need to be compiled together with this module in the multi-platform mode.
Should be a subset of sources passed as free arguments
-Xcoroutines={enable|warn|error}
Enable coroutines or report warnings or errors on declarations and use sites of 'suspend' modifier
-Xdump-perf=<path> Dump detailed performance statistics to the specified file
@@ -42,6 +42,7 @@ import org.jetbrains.kotlin.builtins.KotlinBuiltIns;
import org.jetbrains.kotlin.checkers.CompilerTestLanguageVersionSettings;
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys;
import org.jetbrains.kotlin.cli.common.config.ContentRootsKt;
import org.jetbrains.kotlin.cli.common.config.KotlinSourceRoot;
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation;
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity;
import org.jetbrains.kotlin.cli.common.messages.MessageCollector;
@@ -610,11 +611,11 @@ public class KotlinTestUtils {
}
public static void resolveAllKotlinFiles(KotlinCoreEnvironment environment) throws IOException {
List<String> paths = ContentRootsKt.getKotlinSourceRoots(environment.getConfiguration());
if (paths.isEmpty()) return;
List<KotlinSourceRoot> roots = ContentRootsKt.getKotlinSourceRoots(environment.getConfiguration());
if (roots.isEmpty()) return;
List<KtFile> ktFiles = new ArrayList<>();
for (String path : paths) {
File file = new File(path);
for (KotlinSourceRoot root : roots) {
File file = new File(root.getPath());
if (file.isFile()) {
ktFiles.add(loadJetFile(environment.getProject(), file));
}
@@ -55,24 +55,24 @@ abstract class AbstractMultiPlatformIntegrationTest : KtUsefulTestCase() {
val result = buildString {
appendln("-- Common --")
appendln(K2MetadataCompiler().compile(listOf(commonSrc), "-d", commonDest, *optionalStdlibCommon))
appendln(K2MetadataCompiler().compile(commonSrc, null, "-d", commonDest, *optionalStdlibCommon))
if (jvmSrc != null) {
appendln()
appendln("-- JVM --")
appendln(K2JVMCompiler().compileBothWays(commonSrc, jvmSrc, "-d", jvmDest!!))
appendln(K2JVMCompiler().compile(jvmSrc, commonSrc, "-d", jvmDest!!))
}
if (jsSrc != null) {
appendln()
appendln("-- JS --")
appendln(K2JSCompiler().compileBothWays(commonSrc, jsSrc, "-output", jsDest!!))
appendln(K2JSCompiler().compile(jsSrc, commonSrc, "-output", jsDest!!))
}
if (common2Src != null) {
appendln()
appendln("-- Common (2) --")
appendln(K2MetadataCompiler().compile(listOf(common2Src), "-d", common2Dest!!, "-cp", commonDest, *optionalStdlibCommon))
appendln(K2MetadataCompiler().compile(common2Src, null, "-d", common2Dest!!, "-cp", commonDest, *optionalStdlibCommon))
}
if (jvm2Src != null) {
@@ -80,7 +80,7 @@ abstract class AbstractMultiPlatformIntegrationTest : KtUsefulTestCase() {
appendln("-- JVM (2) --")
appendln(
K2JVMCompiler().compile(
listOf(jvm2Src), "-d", jvm2Dest!!,
jvm2Src, common2Src, "-d", jvm2Dest!!,
"-cp", listOfNotNull(commonDest, common2Dest, jvmDest).joinToString(File.pathSeparator)
)
)
@@ -100,28 +100,12 @@ abstract class AbstractMultiPlatformIntegrationTest : KtUsefulTestCase() {
}?.toFile() ?: error("kotlin-stdlib-common is not found in $stdlibCommonLibsDir")
}
private fun CLICompiler<*>.compileBothWays(commonSource: File, platformSource: File, vararg mainArguments: String): String {
val configurations = listOf(
listOf(platformSource, commonSource),
listOf(commonSource, platformSource)
)
val (platformFirst, commonFirst) = configurations.map { compile(it, *mainArguments) }
if (platformFirst != commonFirst) {
assertEquals(
"Compilation results are different when compiling [platform-specific, common] compared to when compiling [common, platform-specific]",
"// Compiling [platform-specific, common]\n\n$platformFirst",
"// Compiling [common, platform-specific]\n\n$commonFirst"
)
}
return platformFirst
}
private fun CLICompiler<*>.compile(sources: List<File>, vararg mainArguments: String): String = buildString {
private fun CLICompiler<*>.compile(sources: File, commonSources: File?, vararg mainArguments: String): String = buildString {
val (output, exitCode) = AbstractCliTest.executeCompilerGrabOutput(
this@compile,
sources.map(File::getAbsolutePath) + listOf("-Xmulti-platform") + mainArguments + loadExtraArguments(sources)
listOfNotNull(sources.absolutePath, commonSources?.absolutePath, commonSources?.absolutePath?.let("-Xcommon-sources="::plus)) +
"-Xmulti-platform" + mainArguments +
loadExtraArguments(listOfNotNull(sources, commonSources))
)
appendln("Exit code: $exitCode")
appendln("Output:")
@@ -27,6 +27,8 @@ interface Module {
fun getSourceFiles(): List<String>
fun getCommonSourceFiles(): List<String>
fun getClasspathRoots(): List<String>
fun getJavaSourceRoots(): List<JavaRootPath>