From eda29a48a4e7d8499ee7b67693935bef0da0f8ac Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Mon, 27 Aug 2018 17:21:04 +0300 Subject: [PATCH] [K/N lib] IDEA plugin for K/N + missed items in konan-serializer module --- idea/idea-gradle/native/build.gradle.kts | 35 +++ .../konan/gradle/GradleKonanModelProvider.kt | 81 +++++ .../gradle/GradleKonanProjectComponent.kt | 17 ++ .../konan/gradle/KonanProjectDataService.kt | 75 +++++ .../konan/gradle/KonanProjectResolver.kt | 157 ++++++++++ .../internal/KotlinNativeIdeInitializer.java | 33 ++ idea/idea-native/build.gradle.kts | 25 ++ .../konan/KonanApplicationComponent.kt | 27 ++ .../jetbrains/konan/KonanDecompiledFile.kt | 14 + .../konan/KonanPluginSearchPathResolver.kt | 78 +++++ .../org/jetbrains/konan/KonanPluginUtil.kt | 95 ++++++ .../jetbrains/konan/KotlinNativeToolchain.kt | 96 ++++++ .../konan/SetupKotlinNativeStartupActivity.kt | 24 ++ .../konan/analyser/KonanAnalyzerFacade.kt | 84 ++++++ .../konan/analyser/KonanPlatformSupport.kt | 67 +++++ .../analyser/index/KonanDescriptorManager.kt | 47 +++ .../konan/analyser/index/KonanMetaBinary.java | 14 + .../analyser/index/KonanMetaFileIndex.kt | 33 ++ .../analyser/index/KonanMetadataDecompiler.kt | 55 ++++ .../index/KonanMetadataDecompilerBase.kt | 121 ++++++++ .../KonanMetadataDeserializerForDecompiler.kt | 72 +++++ .../index/KonanMetadataStubBuilder.kt | 47 +++ .../index/KonanProtoBasedClassDataFinder.kt | 33 ++ .../jetbrains/konan/settings/KonanArtifact.kt | 25 ++ .../konan/settings/KonanModelProvider.java | 31 ++ .../jetbrains/konan/settings/KonanPaths.kt | 53 ++++ .../konan/settings/KonanProjectComponent.kt | 104 +++++++ idea/src/META-INF/gradle.xml | 27 ++ idea/src/META-INF/native.xml | 37 +++ idea/src/META-INF/plugin.xml | 1 + idea/src/META-INF/plugin.xml.183 | 1 + idea/src/META-INF/plugin.xml.as32 | 1 + idea/src/META-INF/plugin.xml.as33 | 1 + ...onanDeserializedModuleDescriptorFactory.kt | 7 +- .../KonanResolvedModuleDescriptorsFactory.kt | 4 +- ...DeserializedModuleDescriptorFactoryImpl.kt | 24 +- ...nanResolvedModuleDescriptorsFactoryImpl.kt | 27 +- .../konan/library/KonanLibraryConstants.kt | 26 ++ .../konan/library/SearchPathResolver.kt | 7 +- .../kotlin/konan/util/DependencyDownloader.kt | 216 +++++++++++++ .../kotlin/konan/util/DependencyExtractor.kt | 38 +++ .../kotlin/konan/util/DependencyProcessor.kt | 284 ++++++++++++++++++ prepare/idea-plugin/build.gradle.kts | 5 + settings.gradle | 2 + 44 files changed, 2230 insertions(+), 21 deletions(-) create mode 100644 idea/idea-gradle/native/build.gradle.kts create mode 100644 idea/idea-gradle/native/src/org/jetbrains/konan/gradle/GradleKonanModelProvider.kt create mode 100644 idea/idea-gradle/native/src/org/jetbrains/konan/gradle/GradleKonanProjectComponent.kt create mode 100644 idea/idea-gradle/native/src/org/jetbrains/konan/gradle/KonanProjectDataService.kt create mode 100644 idea/idea-gradle/native/src/org/jetbrains/konan/gradle/KonanProjectResolver.kt create mode 100644 idea/idea-gradle/native/src/org/jetbrains/konan/gradle/internal/KotlinNativeIdeInitializer.java create mode 100644 idea/idea-native/build.gradle.kts create mode 100644 idea/idea-native/src/org/jetbrains/konan/KonanApplicationComponent.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/KonanDecompiledFile.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/KonanPluginSearchPathResolver.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/KonanPluginUtil.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/KotlinNativeToolchain.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/SetupKotlinNativeStartupActivity.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/analyser/KonanAnalyzerFacade.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/analyser/KonanPlatformSupport.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanDescriptorManager.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetaBinary.java create mode 100644 idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetaFileIndex.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataDecompiler.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataDecompilerBase.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataDeserializerForDecompiler.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataStubBuilder.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanProtoBasedClassDataFinder.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/settings/KonanArtifact.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/settings/KonanModelProvider.java create mode 100644 idea/idea-native/src/org/jetbrains/konan/settings/KonanPaths.kt create mode 100644 idea/idea-native/src/org/jetbrains/konan/settings/KonanProjectComponent.kt create mode 100644 idea/src/META-INF/native.xml create mode 100644 konan/utils/src/org/jetbrains/kotlin/konan/library/KonanLibraryConstants.kt create mode 100644 konan/utils/src/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt create mode 100644 konan/utils/src/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt create mode 100644 konan/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt diff --git a/idea/idea-gradle/native/build.gradle.kts b/idea/idea-gradle/native/build.gradle.kts new file mode 100644 index 00000000000..3e989582682 --- /dev/null +++ b/idea/idea-gradle/native/build.gradle.kts @@ -0,0 +1,35 @@ +plugins { + kotlin("jvm") + id("jps-compatible") +} + +dependencies { + compile(project(":konan:konan-serializer")) + +// compile("org.jetbrains.kotlin:kotlin-native-gradle-plugin") + + compileOnly(project(":idea:idea-gradle")) + compileOnly(project(":idea:idea-native")) + + compileOnly(project(":idea")) { isTransitive = false } + compileOnly(project(":idea:idea-jvm")) + compile(project(":idea:kotlin-gradle-tooling")) + + compile(project(":compiler:frontend")) + compile(project(":compiler:frontend.java")) + compile(project(":compiler:frontend.script")) + + compile(project(":js:js.frontend")) + + compileOnly(intellijDep()) + compileOnly(intellijPluginDep("gradle")) + compileOnly(intellijPluginDep("Groovy")) + compileOnly(intellijPluginDep("junit")) +} + +sourceSets { + "main" { projectDefault() } + "test" { none() } +} + +configureInstrumentation() diff --git a/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/GradleKonanModelProvider.kt b/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/GradleKonanModelProvider.kt new file mode 100644 index 00000000000..5a5cb6baa15 --- /dev/null +++ b/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/GradleKonanModelProvider.kt @@ -0,0 +1,81 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.gradle + +import com.intellij.openapi.externalSystem.model.ProjectKeys +import com.intellij.openapi.externalSystem.service.project.ProjectDataManager +import com.intellij.openapi.externalSystem.util.ExternalSystemApiUtil.find +import com.intellij.openapi.externalSystem.util.ExternalSystemApiUtil.findAll +import com.intellij.openapi.project.Project +import com.intellij.openapi.util.Key +import com.intellij.util.BooleanFunction +import com.sun.media.jfxmediaimpl.platform.PlatformManager +import org.jetbrains.konan.settings.KonanArtifact +import org.jetbrains.konan.settings.KonanModelProvider +import org.jetbrains.kotlin.gradle.plugin.model.KonanModel +import org.jetbrains.kotlin.gradle.plugin.model.KonanModelArtifact +import org.jetbrains.kotlin.konan.target.PlatformManager +import org.jetbrains.kotlin.konan.target.customerDistribution +import org.jetbrains.plugins.gradle.settings.GradleSettings +import org.jetbrains.plugins.gradle.util.GradleConstants +import java.io.File +import java.nio.file.Path + +class GradleKonanModelProvider : KonanModelProvider { + override fun reloadLibraries(project: Project, libraryPaths: Collection): Boolean = + GradleSettings.getInstance(project).linkedProjectsSettings.isNotEmpty() + + override fun getKonanHome(project: Project): Path? { + val projectNode = ProjectDataManager.getInstance().getExternalProjectsData(project, GradleConstants.SYSTEM_ID) + .mapNotNull { it.externalProjectStructure } + .firstOrNull() ?: return null + projectNode.getUserData(KONAN_HOME)?.let { return it } + + var konanHomePath: Path? = null + find(projectNode, ProjectKeys.MODULE, BooleanFunction { moduleNode -> + find(moduleNode, KonanProjectResolver.KONAN_MODEL_KEY, BooleanFunction { konanModelNode -> + konanHomePath = konanModelNode.data.konanHome.toPath() + projectNode.putUserData(KONAN_HOME, konanHomePath) + konanHomePath != null + }) != null + }) + return konanHomePath + } + + override fun getArtifacts(project: Project): Collection { + val artifacts = mutableListOf() + ProjectDataManager.getInstance().getExternalProjectsData(project, GradleConstants.SYSTEM_ID) + .mapNotNull { it.externalProjectStructure } + .forEach { projectStructure -> + findAll(projectStructure, ProjectKeys.MODULE) + .map { Pair(it.data.externalName, find(it, KonanProjectResolver.KONAN_MODEL_KEY)) } + .filter { it.second != null } + .forEach { (moduleName, konanProjectNode) -> + konanProjectNode!!.data.artifacts.forEach { konanArtifact -> + val sources = konanArtifact.srcFiles.map { it.toPath() } + + artifacts.add( + KonanArtifact( + konanArtifact.name, + moduleName, + konanArtifact.type, + konanTarget(konanProjectNode.data.konanHome, konanArtifact), + mutableListOf(), sources, konanArtifact.file.toPath() + ) + ) + } + } + } + return artifacts + } + + private fun konanTarget(konanHome: File, konanArtifactEx: KonanModelArtifact) = + PlatformManager(customerDistribution(konanHome.absolutePath)).targetValues.find { it.name == konanArtifactEx.targetPlatform } + + companion object { + val KONAN_HOME = Key.create("KONAN_HOME") + } +} diff --git a/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/GradleKonanProjectComponent.kt b/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/GradleKonanProjectComponent.kt new file mode 100644 index 00000000000..1389a076339 --- /dev/null +++ b/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/GradleKonanProjectComponent.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.gradle + +import com.intellij.openapi.project.Project +import org.jetbrains.konan.settings.KonanProjectComponent +import org.jetbrains.plugins.gradle.settings.GradleSettings + +class GradleKonanProjectComponent(project: Project) : KonanProjectComponent(project) { + override fun looksLikeKotlinNativeProject(): Boolean { + //TODO not just any gradle project + return GradleSettings.getInstance(project).linkedProjectsSettings.isNotEmpty() + } +} \ No newline at end of file diff --git a/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/KonanProjectDataService.kt b/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/KonanProjectDataService.kt new file mode 100644 index 00000000000..1865e6dc33c --- /dev/null +++ b/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/KonanProjectDataService.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.gradle + +import com.intellij.execution.RunManager +import com.intellij.execution.RunnerAndConfigurationSettings +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.externalSystem.model.DataNode +import com.intellij.openapi.externalSystem.model.Key +import com.intellij.openapi.externalSystem.model.ProjectKeys +import com.intellij.openapi.externalSystem.model.project.ModuleData +import com.intellij.openapi.externalSystem.model.project.ProjectData +import com.intellij.openapi.externalSystem.service.project.IdeModelsProvider +import com.intellij.openapi.externalSystem.service.project.IdeModifiableModelsProvider +import com.intellij.openapi.externalSystem.service.project.ProjectDataManager +import com.intellij.openapi.externalSystem.service.project.manage.AbstractProjectDataService +import com.intellij.openapi.externalSystem.util.ExternalSystemApiUtil +import com.intellij.openapi.module.Module +import com.intellij.openapi.project.Project +import com.intellij.util.containers.ContainerUtil +import org.jetbrains.konan.settings.KonanArtifact +import org.jetbrains.konan.settings.KonanModelProvider +import org.jetbrains.konan.settings.isExecutable +import org.jetbrains.kotlin.gradle.plugin.model.KonanModel +import org.jetbrains.plugins.gradle.util.GradleConstants + +class KonanProjectDataService : AbstractProjectDataService() { + + override fun getTargetDataKey(): Key = KonanProjectResolver.KONAN_MODEL_KEY + + override fun postProcess( + toImport: Collection>, + projectData: ProjectData?, + project: Project, + modelsProvider: IdeModifiableModelsProvider + ) { + } + + override fun onSuccessImport( + imported: Collection>, + projectData: ProjectData?, + project: Project, + modelsProvider: IdeModelsProvider + ) { + if (projectData?.owner != GradleConstants.SYSTEM_ID) return + project.messageBus.syncPublisher(KonanModelProvider.RELOAD_TOPIC).run() + } + + companion object { + @JvmStatic + fun forEachKonanProject( + project: Project, + consumer: (konanProject: KonanModel, moduleData: ModuleData, rootProjectPath: String) -> Unit + ) { + for (projectInfo in ProjectDataManager.getInstance().getExternalProjectsData(project, GradleConstants.SYSTEM_ID)) { + val projectStructure = projectInfo.externalProjectStructure ?: continue + val projectData = projectStructure.data + val rootProjectPath = projectData.linkedExternalProjectPath + val modulesNodes = ExternalSystemApiUtil.findAll(projectStructure, ProjectKeys.MODULE) + for (moduleNode in modulesNodes) { + val projectNode = ExternalSystemApiUtil.find(moduleNode, KonanProjectResolver.KONAN_MODEL_KEY) + if (projectNode != null) { + val konanProject = projectNode.data + val moduleData = moduleNode.data + consumer(konanProject, moduleData, rootProjectPath) + } + } + } + } + } + +} diff --git a/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/KonanProjectResolver.kt b/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/KonanProjectResolver.kt new file mode 100644 index 00000000000..cb71c1203a0 --- /dev/null +++ b/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/KonanProjectResolver.kt @@ -0,0 +1,157 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.gradle + +import com.intellij.openapi.diagnostic.Logger +import com.intellij.openapi.externalSystem.model.DataNode +import com.intellij.openapi.externalSystem.model.Key +import com.intellij.openapi.externalSystem.model.ProjectKeys +import com.intellij.openapi.externalSystem.model.project.* +import com.intellij.util.containers.ContainerUtil.set +import com.intellij.util.io.isDirectory +import org.gradle.tooling.model.idea.IdeaModule +import org.jetbrains.kotlin.gradle.plugin.model.KonanModel +import org.jetbrains.kotlin.gradle.plugin.model.KonanModelArtifact +import org.jetbrains.kotlin.konan.KonanVersion +import org.jetbrains.kotlin.konan.MetaVersion +import org.jetbrains.kotlin.konan.target.CompilerOutputKind +import org.jetbrains.kotlin.konan.target.HostManager +import org.jetbrains.plugins.gradle.service.project.AbstractProjectResolverExtension +import org.jetbrains.plugins.gradle.util.GradleConstants +import java.io.File +import java.nio.file.Files.isDirectory +import java.nio.file.Files.walk +import java.nio.file.Path +import java.util.* + +/** + * [KonanProjectResolver] creates IDE project model in terms of External System API + */ +class KonanProjectResolver : AbstractProjectResolverExtension() { + + // to ask gradle for the model + override fun getExtraProjectModelClasses(): Set> { + return set>(KonanModel::class.java) + } + + override fun populateModuleExtraModels(gradleModule: IdeaModule, ideModule: DataNode) { + resolverCtx.getExtraProject(gradleModule, KonanModel::class.java)?.let { + // store a local process copy of the object to get rid of proxy types for further serialization + ideModule.createChild(KONAN_MODEL_KEY, MyKonanModel(it)) + } + + nextResolver.populateModuleExtraModels(gradleModule, ideModule) + } + + override fun populateModuleContentRoots(gradleModule: IdeaModule, ideModule: DataNode) { + resolverCtx.getExtraProject(gradleModule, KonanModel::class.java)?.let { + val added = mutableSetOf() + for (artifact in it.artifacts) { + for (srcDir in artifact.srcDirs) { + val rootPath = srcDir.absolutePath + if (!added.add(rootPath)) continue + + val ideContentRoot = ContentRootData(GradleConstants.SYSTEM_ID, rootPath) + ideContentRoot.storePath(ExternalSystemSourceType.SOURCE, rootPath) + ideModule.createChild(ProjectKeys.CONTENT_ROOT, ideContentRoot) + } + } + } + + nextResolver.populateModuleContentRoots(gradleModule, ideModule) + } + + // based on KonanCMakeProjectComponent.reloadLibraries but with the multi-module projects support + override fun populateModuleDependencies( + gradleModule: IdeaModule, + ideModule: DataNode, + ideProject: DataNode + ) { + val konanModelEx = resolverCtx.getExtraProject(gradleModule, KonanModel::class.java) + if (konanModelEx != null) { + val libraryPaths = LinkedHashSet() + var konanHome: Path? = null + var targetPlatform: String? = null + for (konanArtifact in konanModelEx.artifacts) { + if (konanHome == null) { + konanHome = konanModelEx.konanHome.toPath() + targetPlatform = konanArtifact.targetPlatform + } + konanArtifact.libraries.forEach { libraryPaths.add(it.toPath()) } + } + + if (konanHome != null) { + // add konanStdlib copied from KonanPaths.konanStdlib + libraryPaths.add(konanHome.resolve("klib/common/stdlib")) + + // add konanPlatformLibraries, copied from KonanPaths.konanPlatformLibraries + if (targetPlatform != null) { + try { + val resolvedTargetName = HostManager.resolveAlias(targetPlatform) + val klibPath = konanHome.resolve("klib/platform/${resolvedTargetName}") + walk(klibPath, 1) + .filter { it.isDirectory() && it.fileName.toString() != "stdlib" && it != klibPath } + .forEach { libraryPaths.add(it) } + } catch (e: Exception) { + LOG.warn("Unable to collect konan platform libraries paths for '$targetPlatform'", e) + } + } + } + + val moduleData = ideModule.data + for (path in libraryPaths) { + val library = LibraryData(moduleData.owner, path.fileName.toString()) + library.addPath(LibraryPathType.BINARY, if (isDirectory(path)) path.toAbsolutePath().toString() else path.toString()) + + val data = LibraryDependencyData(moduleData, library, LibraryLevel.MODULE) + ideModule.createChild(ProjectKeys.LIBRARY_DEPENDENCY, data) + } + } + + nextResolver.populateModuleDependencies(gradleModule, ideModule, ideProject) + } + + private class MyKonanModel(konanModel: KonanModel) : KonanModel { + override val artifacts: List = konanModel.artifacts.map { MyKonanArtifactEx(it) } + override val konanHome: File = konanModel.konanHome + override val konanVersion: KonanVersion = MyKonanVersionEx(konanModel.konanVersion) + override val apiVersion: String? = konanModel.apiVersion + override val languageVersion: String? = konanModel.languageVersion + + private class MyKonanVersionEx(version: KonanVersion) : KonanVersion { + override val build: Int = version.build + override val maintenance: Int = version.maintenance + override val major: Int = version.major + override val meta: MetaVersion = version.meta + override val minor: Int = version.minor + + override fun toString(showMeta: Boolean, showBuild: Boolean): String { + val sb = StringBuilder("$major.$minor.$maintenance") + if (showMeta) sb.append('-').append(meta) + if (showBuild) sb.append('-').append(build) + return sb.toString() + } + } + + private class MyKonanArtifactEx(artifact: KonanModelArtifact) : KonanModelArtifact { + override val searchPaths: List = artifact.searchPaths + override val name: String = artifact.name + override val type: CompilerOutputKind = artifact.type + override val targetPlatform: String = artifact.targetPlatform + override val file: File = artifact.file + override val buildTaskName: String = artifact.buildTaskName + + override val srcDirs: List = artifact.srcDirs.toList() + override val srcFiles: List = artifact.srcFiles.toList() + override val libraries: List = artifact.libraries.toList() + } + } + + companion object { + val KONAN_MODEL_KEY = Key.create(KonanModel::class.java, ProjectKeys.MODULE.processingWeight + 1) + private val LOG = Logger.getInstance(KonanProjectResolver::class.java) + } +} diff --git a/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/internal/KotlinNativeIdeInitializer.java b/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/internal/KotlinNativeIdeInitializer.java new file mode 100644 index 00000000000..9449b9da21c --- /dev/null +++ b/idea/idea-gradle/native/src/org/jetbrains/konan/gradle/internal/KotlinNativeIdeInitializer.java @@ -0,0 +1,33 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.gradle.internal; + +import com.intellij.codeInspection.LocalInspectionEP; +import com.intellij.openapi.components.ApplicationComponent; +import com.intellij.openapi.extensions.ExtensionPoint; +import com.intellij.openapi.extensions.Extensions; + +public class KotlinNativeIdeInitializer implements ApplicationComponent { + + @Override + public void initComponent() { + unregisterGroovyInspections(); + } + + // There are groovy local inspections which should not be loaded w/o groovy plugin enabled. + // Those plugin definitions should become optional and dependant on groovy plugin. + // This is a temp workaround before it happens. + private static void unregisterGroovyInspections() { + ExtensionPoint extensionPoint = + Extensions.getRootArea().getExtensionPoint(LocalInspectionEP.LOCAL_INSPECTION); + + for (LocalInspectionEP ep : extensionPoint.getExtensions()) { + if ("Kotlin".equals(ep.groupDisplayName) && "Groovy".equals(ep.language)) { + extensionPoint.unregisterExtension(ep); + } + } + } +} diff --git a/idea/idea-native/build.gradle.kts b/idea/idea-native/build.gradle.kts new file mode 100644 index 00000000000..2e911a7bedd --- /dev/null +++ b/idea/idea-native/build.gradle.kts @@ -0,0 +1,25 @@ +plugins { + kotlin("jvm") +} + +dependencies { + compile(project(":idea")) + compile(project(":idea:idea-core")) + compile(project(":compiler:frontend")) + compileOnly(intellijDep()) + compile(project(":konan:konan-serializer")) +} + +sourceSets { + "main" { + projectDefault() +// java.srcDirs("$rootDir/core/runtime.jvm/src") + } + "test" { none() } +} + +configureInstrumentation() + +runtimeJar { + archiveName = "native-ide.jar" +} diff --git a/idea/idea-native/src/org/jetbrains/konan/KonanApplicationComponent.kt b/idea/idea-native/src/org/jetbrains/konan/KonanApplicationComponent.kt new file mode 100644 index 00000000000..e78c6701e93 --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/KonanApplicationComponent.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan + +import com.intellij.ide.highlighter.ArchiveFileType +import com.intellij.ide.util.TipAndTrickBean +import com.intellij.openapi.components.ApplicationComponent +import com.intellij.openapi.extensions.Extensions +import com.intellij.openapi.fileTypes.FileTypeManager + +class KonanApplicationComponent : ApplicationComponent { + override fun getComponentName(): String = "KonanApplicationComponent" + + override fun initComponent() { + FileTypeManager.getInstance().associateExtension(ArchiveFileType.INSTANCE, "klib") + + val extensionPoint = Extensions.getRootArea().getExtensionPoint(TipAndTrickBean.EP_NAME) + for (name in arrayOf("Kotlin.html", "Kotlin_project.html", "Kotlin_mix.html", "Kotlin_Java_convert.html")) { + TipAndTrickBean.findByFileName(name)?.let { + extensionPoint.unregisterExtension(it) + } + } + } +} diff --git a/idea/idea-native/src/org/jetbrains/konan/KonanDecompiledFile.kt b/idea/idea-native/src/org/jetbrains/konan/KonanDecompiledFile.kt new file mode 100644 index 00000000000..a09e4a6e36a --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/KonanDecompiledFile.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan + +import com.intellij.openapi.vfs.VirtualFile +import org.jetbrains.kotlin.idea.decompiler.KotlinDecompiledFileViewProvider +import org.jetbrains.kotlin.idea.decompiler.KtDecompiledFile +import org.jetbrains.kotlin.idea.decompiler.textBuilder.DecompiledText + +class KonanDecompiledFile(provider: KotlinDecompiledFileViewProvider, text: (VirtualFile) -> DecompiledText) : + KtDecompiledFile(provider, text) diff --git a/idea/idea-native/src/org/jetbrains/konan/KonanPluginSearchPathResolver.kt b/idea/idea-native/src/org/jetbrains/konan/KonanPluginSearchPathResolver.kt new file mode 100644 index 00000000000..37a8a8a3f7c --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/KonanPluginSearchPathResolver.kt @@ -0,0 +1,78 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan + +import com.intellij.openapi.project.Project +import org.jetbrains.konan.settings.KonanPaths +import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.konan.library.* +import org.jetbrains.kotlin.konan.target.KonanTarget + +class KonanPluginSearchPathResolver(val project: Project) : SearchPathResolverWithTarget { + + private val paths by lazy { KonanPaths(project) } + + override val searchRoots: List by lazy { + paths.konanDist()?.let { distPath -> + listOf(KONAN_COMMON_LIBS_PATH, konanSpecificPlatformLibrariesPath(target.toString())).mapNotNull { relativePath -> + distPath.resolve(relativePath).File().takeIf { it.exists } + } + } ?: emptyList() + } + + override fun resolve(givenPath: String): File { + + val given = File(givenPath) + + if (given.isAbsolute) { + found(given)?.apply { return this } + } else { + searchRoots.forEach { + found(File(it, givenPath))?.apply { return this } + } + } + + error("Could not find \"$givenPath\" in ${searchRoots.map { it.absolutePath }}.") + } + + override fun defaultLinks(noStdLib: Boolean, noDefaultLibs: Boolean): List { + + val result = mutableListOf() + + if (!noStdLib) { + result.add(resolve(KONAN_STDLIB_NAME)) + } + + if (!noDefaultLibs) { + val defaultLibs = searchRoots.flatMap { it.listFiles } + .filterNot { it.name.removeSuffix(KLIB_FILE_EXTENSION_WITH_DOT) == KONAN_STDLIB_NAME } + .map { File(it.absolutePath) } + result.addAll(defaultLibs) + } + + return result + } + + override val target: KonanTarget + get() = paths.target() + + private fun found(candidate: File): File? { + + fun check(file: File): Boolean = + file.exists && (file.isFile || File(file, "manifest").exists) + + val noSuffix = File(candidate.path.removeSuffix(KLIB_FILE_EXTENSION_WITH_DOT)) + val withSuffix = File(candidate.path.suffixIfNot(KLIB_FILE_EXTENSION_WITH_DOT)) + + return when { + check(withSuffix) -> withSuffix + check(noSuffix) -> noSuffix + else -> null + } + } +} + +private fun String.suffixIfNot(suffix: String) = if (this.endsWith(suffix)) this else "$this$suffix" diff --git a/idea/idea-native/src/org/jetbrains/konan/KonanPluginUtil.kt b/idea/idea-native/src/org/jetbrains/konan/KonanPluginUtil.kt new file mode 100644 index 00000000000..e923ffacbab --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/KonanPluginUtil.kt @@ -0,0 +1,95 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan + +import com.intellij.openapi.diagnostic.Logger +import com.intellij.openapi.module.Module +import com.intellij.openapi.project.Project +import com.intellij.openapi.roots.ModuleRootManager +import com.intellij.psi.PsiFileFactory +import com.intellij.psi.SingleRootFileViewProvider +import com.intellij.psi.impl.PsiFileFactoryImpl +import com.intellij.psi.search.GlobalSearchScope +import com.intellij.psi.stubs.PsiFileStub +import com.intellij.testFramework.LightVirtualFile +import org.jetbrains.kotlin.analyzer.ModuleContent +import org.jetbrains.kotlin.analyzer.ModuleInfo +import org.jetbrains.kotlin.builtins.KotlinBuiltIns +import org.jetbrains.kotlin.config.LanguageVersionSettings +import org.jetbrains.kotlin.context.ModuleContext +import org.jetbrains.kotlin.idea.KotlinFileType +import org.jetbrains.kotlin.idea.KotlinLanguage +import org.jetbrains.kotlin.idea.caches.project.LibraryInfo +import org.jetbrains.kotlin.idea.decompiler.textBuilder.LoggingErrorReporter +import org.jetbrains.kotlin.konan.library.KonanLibrary +import org.jetbrains.kotlin.konan.library.libraryResolver +import org.jetbrains.kotlin.konan.utils.KonanFactories +import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.psi.stubs.elements.KtStubElementTypes +import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactoryService +import org.jetbrains.kotlin.serialization.konan.KonanResolvedModuleDescriptors +import org.jetbrains.kotlin.storage.StorageManager +import java.io.File.pathSeparatorChar + +const val KONAN_CURRENT_ABI_VERSION = 1 + +fun createFileStub(project: Project, text: String): PsiFileStub<*> { + val virtualFile = LightVirtualFile("dummy.kt", KotlinFileType.INSTANCE, text) + virtualFile.language = KotlinLanguage.INSTANCE + SingleRootFileViewProvider.doNotCheckFileSizeLimit(virtualFile) + + val psiFileFactory = PsiFileFactory.getInstance(project) as PsiFileFactoryImpl + val file = psiFileFactory.trySetupPsiForFile(virtualFile, KotlinLanguage.INSTANCE, false, false)!! + return KtStubElementTypes.FILE.builder.buildStubTree(file) as PsiFileStub<*> +} + +fun createLoggingErrorReporter(log: Logger) = LoggingErrorReporter(log) + +fun destructModuleContent(moduleContent: ModuleContent) = + moduleContent.syntheticFiles to moduleContent.moduleContentScope + +fun createDeclarationProviderFactory( + project: Project, + moduleContext: ModuleContext, + syntheticFiles: Collection, + moduleInfo: M, + globalSearchScope: GlobalSearchScope? +) = DeclarationProviderFactoryService.createDeclarationProviderFactory( + project, + moduleContext.storageManager, + syntheticFiles, + globalSearchScope!!, + moduleInfo +) + +fun Module.createResolvedModuleDescriptors( + storageManager: StorageManager, + builtIns: KotlinBuiltIns, + languageVersionSettings: LanguageVersionSettings +): KonanResolvedModuleDescriptors { + + val libraryMap = mutableMapOf() + ModuleRootManager.getInstance(this).orderEntries().forEachLibrary { intellijLibrary -> + intellijLibrary.name?.let { name -> libraryMap[name] = LibraryInfo(project, intellijLibrary) } + true + } + + val resolvedLibraries = + KonanPluginSearchPathResolver(project).libraryResolver(KONAN_CURRENT_ABI_VERSION).resolveWithDependencies(libraryMap.keys.toList()) + + return KonanFactories.DefaultResolvedDescriptorsFactory.createResolved( + resolvedLibraries, + storageManager, + builtIns, + languageVersionSettings, + null, + // Preserve capabilities from the original IntelliJ library: + { konanLibrary -> libraryMap[konanLibrary.pureName]?.capabilities ?: emptyMap() } + ) +} + +private val KonanLibrary.pureName + get() = libraryName.substringAfterLast(pathSeparatorChar) diff --git a/idea/idea-native/src/org/jetbrains/konan/KotlinNativeToolchain.kt b/idea/idea-native/src/org/jetbrains/konan/KotlinNativeToolchain.kt new file mode 100644 index 00000000000..58a9c2b4fc2 --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/KotlinNativeToolchain.kt @@ -0,0 +1,96 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan + +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.diagnostic.Logger +import com.intellij.openapi.progress.ProgressIndicator +import com.intellij.openapi.progress.Task +import com.intellij.openapi.project.Project +import com.intellij.util.SystemProperties.getUserHome +import com.intellij.util.io.exists +import com.intellij.util.net.IOExceptionDialog +import org.jetbrains.konan.settings.KonanProjectComponent +import org.jetbrains.kotlin.konan.target.HostManager +import org.jetbrains.kotlin.konan.util.DependencyDownloaderHTTPResponseException +import org.jetbrains.kotlin.konan.util.DependencyProcessor +import org.jetbrains.kotlin.konan.util.DependencySource +import java.nio.file.Path +import java.nio.file.Paths + +class KotlinNativeToolchain( + val version: String, + val repoUrl: String +) { + private val artifactName = "kotlin-native-$KONAN_OS-$version" + val baseDir: Path get() = Paths.get("${getUserHome()}/.konan/$artifactName") + val konanc: Path get() = baseDir.resolve("bin/konanc") + val cinterop: Path get() = baseDir.resolve("bin/cinterop") + + + fun ensureExists(project: Project) { + ApplicationManager.getApplication().assertIsDispatchThread() + if (baseDir.exists()) return + + for (attempt in 0..3) { + var exception: Throwable? = null + object : Task.Modal(project, "Downloading Kotlin/Native $version", false) { + override fun run(progress: ProgressIndicator) { + DependencyProcessor( + baseDir.parent.toFile(), + repoUrl, + mapOf(artifactName to listOf(DependencySource.Remote.Public)), + customProgressCallback = { _, downloaded, total -> + progress.fraction = downloaded / maxOf(total, downloaded, 1).toDouble() + } + ).run() + ApplicationManager.getApplication().invokeLater { + project.getComponent(KonanProjectComponent::class.java).reloadLibraries() + } + } + + override fun onThrowable(error: Throwable) { + exception = error + } + }.queue() + + val ex = exception + val tryAgain = if (ex == null) { + false + } else { + val details = if (ex is DependencyDownloaderHTTPResponseException) { + "Server returned ${ex.responseCode} when trying to download ${ex.url}." + } else { + LOG.error(ex) + "Unknown error occurred." + } + IOExceptionDialog.showErrorDialog( + "Failed to download Kotlin/Native", + details + ) + } + if (!tryAgain) break + } + } + + companion object { + fun looksLikeBundledToolchain(path: String): Boolean = + Paths.get(path).startsWith(Paths.get("${getUserHome()}/.konan/")) + + private val KONAN_OS = HostManager.simpleOsName() + + //todo: fixme + private val BUNDLED_VERSION = "0.8" //bundledFile("kotlin-native-version").readText().trim() + private val BUILD_DIR = "releases" + + val BUNDLED = KotlinNativeToolchain( + version = BUNDLED_VERSION, + repoUrl = "https://download.jetbrains.com/kotlin/native/builds/$BUILD_DIR/$BUNDLED_VERSION/$KONAN_OS" + ) + + private val LOG = Logger.getInstance(KotlinNativeToolchain::class.java) + } +} diff --git a/idea/idea-native/src/org/jetbrains/konan/SetupKotlinNativeStartupActivity.kt b/idea/idea-native/src/org/jetbrains/konan/SetupKotlinNativeStartupActivity.kt new file mode 100644 index 00000000000..4e32fa2d6dc --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/SetupKotlinNativeStartupActivity.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan + +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.project.Project +import com.intellij.openapi.startup.StartupActivity +import org.jetbrains.konan.settings.KonanProjectComponent + +class SetupKotlinNativeStartupActivity : StartupActivity { + override fun runActivity(project: Project) { + //todo: looks like without default project component (like disabled Gradle plugin) we will get exception here (that's bad) + if (!KonanProjectComponent.getInstance(project).looksLikeKotlinNativeProject()) return + ensureKotlinNativeExists(project) + } + + private fun ensureKotlinNativeExists(project: Project) { + ApplicationManager.getApplication().assertIsDispatchThread() + KotlinNativeToolchain.BUNDLED.ensureExists(project) + } +} diff --git a/idea/idea-native/src/org/jetbrains/konan/analyser/KonanAnalyzerFacade.kt b/idea/idea-native/src/org/jetbrains/konan/analyser/KonanAnalyzerFacade.kt new file mode 100644 index 00000000000..879d6a192ca --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/analyser/KonanAnalyzerFacade.kt @@ -0,0 +1,84 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.analyser + +import org.jetbrains.konan.createDeclarationProviderFactory +import org.jetbrains.konan.createResolvedModuleDescriptors +import org.jetbrains.konan.destructModuleContent +import org.jetbrains.kotlin.analyzer.* +import org.jetbrains.kotlin.config.LanguageVersionSettings +import org.jetbrains.kotlin.config.TargetPlatformVersion +import org.jetbrains.kotlin.container.get +import org.jetbrains.kotlin.context.ModuleContext +import org.jetbrains.kotlin.descriptors.impl.CompositePackageFragmentProvider +import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl +import org.jetbrains.kotlin.frontend.di.createContainerForLazyResolve +import org.jetbrains.kotlin.idea.caches.project.ModuleProductionSourceInfo +import org.jetbrains.kotlin.resolve.BindingTraceContext +import org.jetbrains.kotlin.resolve.TargetEnvironment +import org.jetbrains.kotlin.resolve.TargetPlatform +import org.jetbrains.kotlin.resolve.konan.platform.KonanPlatform +import org.jetbrains.kotlin.resolve.lazy.ResolveSession + +/** + * @author Alefas + */ +class KonanAnalyzerFacade : ResolverForModuleFactory() { + override val targetPlatform: TargetPlatform + get() = KonanPlatform + + override fun createResolverForModule( + moduleDescriptor: ModuleDescriptorImpl, + moduleContext: ModuleContext, + moduleContent: ModuleContent, + platformParameters: PlatformAnalysisParameters, + targetEnvironment: TargetEnvironment, + resolverForProject: ResolverForProject, + languageVersionSettings: LanguageVersionSettings, + targetPlatformVersion: TargetPlatformVersion + ): ResolverForModule { + + val (syntheticFiles, moduleContentScope) = destructModuleContent(moduleContent) + val project = moduleContext.project + + val declarationProviderFactory = createDeclarationProviderFactory( + project, + moduleContext, + syntheticFiles, + moduleContent.moduleInfo, + moduleContentScope + ) + + val container = createContainerForLazyResolve( + moduleContext, + declarationProviderFactory, + BindingTraceContext(), + targetPlatform, + TargetPlatformVersion.NoVersion, + targetEnvironment, + languageVersionSettings + ) + + val packageFragmentProvider = container.get().packageFragmentProvider + val module = (moduleContent.moduleInfo as? ModuleProductionSourceInfo)?.module + + val fragmentProviders = mutableListOf(packageFragmentProvider) + + if (module != null) { + + val moduleDescriptors = module.createResolvedModuleDescriptors( + moduleContext.storageManager, + moduleContext.module.builtIns, // FIXME(ddol): investigate: reuse existing builtIns (from KonanPlatformSupport) or create new one + languageVersionSettings + ) + + moduleDescriptors.resolvedDescriptors.mapTo(fragmentProviders) { it.packageFragmentProvider } + fragmentProviders.add(moduleDescriptors.forwardDeclarationsModule.packageFragmentProvider) + } + + return ResolverForModule(CompositePackageFragmentProvider(fragmentProviders), container) + } +} diff --git a/idea/idea-native/src/org/jetbrains/konan/analyser/KonanPlatformSupport.kt b/idea/idea-native/src/org/jetbrains/konan/analyser/KonanPlatformSupport.kt new file mode 100644 index 00000000000..8e3ca688c19 --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/analyser/KonanPlatformSupport.kt @@ -0,0 +1,67 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.analyser + +import com.intellij.openapi.module.Module +import com.intellij.openapi.project.ProjectManager +import com.intellij.openapi.roots.libraries.PersistentLibraryKind +import com.intellij.openapi.vfs.LocalFileSystem +import org.jetbrains.konan.KONAN_CURRENT_ABI_VERSION +import org.jetbrains.konan.settings.KonanPaths +import org.jetbrains.kotlin.analyzer.ResolverForModuleFactory +import org.jetbrains.kotlin.builtins.DefaultBuiltIns +import org.jetbrains.kotlin.builtins.KotlinBuiltIns +import org.jetbrains.kotlin.caches.resolve.IdePlatformSupport +import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl +import org.jetbrains.kotlin.context.GlobalContextImpl +import org.jetbrains.kotlin.idea.caches.resolve.PlatformAnalysisSettings +import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.konan.library.createKonanLibrary +import org.jetbrains.kotlin.konan.utils.KonanFactories.DefaultDeserializedDescriptorFactory +import org.jetbrains.kotlin.resolve.TargetPlatform +import org.jetbrains.kotlin.resolve.konan.platform.KonanPlatform + +/** + * @author Alefas + */ +class KonanPlatformSupport : IdePlatformSupport() { + + override val resolverForModuleFactory: ResolverForModuleFactory + get() = KonanAnalyzerFacade() + + override val libraryKind: PersistentLibraryKind<*>? = null + + override val platform: TargetPlatform + get() = KonanPlatform + + override fun createBuiltIns(settings: PlatformAnalysisSettings, sdkContext: GlobalContextImpl) = createKonanBuiltIns(sdkContext) + + override fun isModuleForPlatform(module: Module) = true +} + +private fun createKonanBuiltIns(sdkContext: GlobalContextImpl): KotlinBuiltIns { + + // TODO: it depends on a random project's stdlib, propagate the actual project here + val stdlibLocation = ProjectManager.getInstance().openProjects.asSequence().mapNotNull { + val stdlibPath = KonanPaths.getInstance(it).konanStdlib() ?: return@mapNotNull null + val stdlibVirtualFile = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(stdlibPath.toFile()) ?: return@mapNotNull null + stdlibPath to stdlibVirtualFile + }.firstOrNull() + + if (stdlibLocation != null) { + val library = createKonanLibrary(stdlibLocation.first.File(), KONAN_CURRENT_ABI_VERSION) + + val builtInsModule = DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns( + library, + LanguageVersionSettingsImpl.DEFAULT, + sdkContext.storageManager + ) + + return builtInsModule.builtIns + } + + return DefaultBuiltIns.Instance +} diff --git a/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanDescriptorManager.kt b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanDescriptorManager.kt new file mode 100644 index 00000000000..dba80606ec4 --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanDescriptorManager.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.analyser.index + +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.components.ApplicationComponent +import com.intellij.openapi.vfs.* +import com.intellij.util.containers.ContainerUtil.createConcurrentWeakValueMap +import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf +import org.jetbrains.kotlin.serialization.konan.parsePackageFragment + +class KonanDescriptorManager : ApplicationComponent { + + companion object { + + private const val currentAbiVersion = 1 + + @JvmStatic + fun getInstance(): KonanDescriptorManager = ApplicationManager.getApplication().getComponent(KonanDescriptorManager::class.java) + } + + private val protoCache = createConcurrentWeakValueMap() + + fun getCachedPackageFragment(virtualFile: VirtualFile): KonanProtoBuf.LinkDataPackageFragment { + return protoCache.computeIfAbsent(virtualFile) { + val bytes = virtualFile.contentsToByteArray(false) + parsePackageFragment(bytes) + } + } + + override fun initComponent() { + VirtualFileManager.getInstance().addVirtualFileListener(object : VirtualFileListener { + override fun fileCreated(event: VirtualFileEvent) = invalidateCaches(event.file) + override fun fileDeleted(event: VirtualFileEvent) = invalidateCaches(event.file) + override fun fileMoved(event: VirtualFileMoveEvent) = invalidateCaches(event.file) + override fun contentsChanged(event: VirtualFileEvent) = invalidateCaches(event.file) + override fun propertyChanged(event: VirtualFilePropertyEvent) = invalidateCaches(event.file) + }) + } + + private fun invalidateCaches(virtualFile: VirtualFile) { + protoCache.remove(virtualFile) + } +} diff --git a/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetaBinary.java b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetaBinary.java new file mode 100644 index 00000000000..839a19e8d02 --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetaBinary.java @@ -0,0 +1,14 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.analyser.index; + +import org.jetbrains.kotlin.idea.util.KotlinBinaryExtension; + +public class KonanMetaBinary extends KotlinBinaryExtension { + public KonanMetaBinary() { + super(KonanMetaFileType.INSTANCE); + } +} diff --git a/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetaFileIndex.kt b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetaFileIndex.kt new file mode 100644 index 00000000000..00d16204ae8 --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetaFileIndex.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.analyser.index + +import com.intellij.util.indexing.FileBasedIndex +import org.jetbrains.kotlin.idea.vfilefinder.KotlinFileIndexBase +import org.jetbrains.kotlin.name.FqName + +class KonanMetaFileIndex + : KotlinFileIndexBase(KonanMetaFileIndex::class.java) { + + companion object { + private const val VERSION = 4 + } + + /*todo: check version?!*/ + private val dataIndexer = indexer { fileContent -> + val fragment = KonanDescriptorManager.getInstance().getCachedPackageFragment(fileContent.file) + FqName(fragment.fqName) + } + + // this is to express intention to index all Kotlin/Native metadata files irrespectively to file size + override fun getFileTypesWithSizeLimitNotApplicable() = listOf(KonanMetaFileType) + + override fun getInputFilter() = FileBasedIndex.InputFilter { it.fileType === KonanMetaFileType } + + override fun getIndexer() = dataIndexer + + override fun getVersion() = VERSION +} diff --git a/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataDecompiler.kt b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataDecompiler.kt new file mode 100644 index 00000000000..9cd48296e45 --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataDecompiler.kt @@ -0,0 +1,55 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.analyser.index + +import com.intellij.openapi.fileTypes.FileType +import com.intellij.openapi.fileTypes.FileTypeConsumer +import com.intellij.openapi.fileTypes.FileTypeFactory +import com.intellij.openapi.vfs.VirtualFile +import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion +import org.jetbrains.kotlin.resolve.konan.platform.KonanPlatform +import org.jetbrains.kotlin.serialization.konan.KonanSerializerProtocol +import org.jetbrains.kotlin.serialization.konan.NullFlexibleTypeDeserializer + +class KonanMetadataDecompiler : KonanMetadataDecompilerBase( + KonanMetaFileType, KonanPlatform, KonanSerializerProtocol, NullFlexibleTypeDeserializer, + KonanMetadataVersion.DEFAULT_INSTANCE, KonanMetadataVersion.INVALID_VERSION, KonanMetaFileType.STUB_VERSION +) { + + override fun doReadFile(file: VirtualFile): FileWithMetadata? { + val proto = KonanDescriptorManager.getInstance().getCachedPackageFragment(file) + return FileWithMetadata.Compatible(proto, KonanSerializerProtocol) //todo: check version compatibility + } +} + +class KonanMetadataVersion(vararg numbers: Int) : BinaryVersion(*numbers) { + override fun isCompatible(): Boolean = true //todo: ? + + companion object { + @JvmField + val DEFAULT_INSTANCE = KonanMetadataVersion(1, 1, 0) + + @JvmField + val INVALID_VERSION = KonanMetadataVersion() + } +} + +object KonanMetaFileType : FileType { + override fun getName() = "KNM" + override fun getDescription() = "Kotlin/Native Metadata" + override fun getDefaultExtension() = "knm" + override fun getIcon() = null + override fun isBinary() = true + override fun isReadOnly() = true + override fun getCharset(file: VirtualFile, content: ByteArray) = null + + const val STUB_VERSION = 2 +} + +class KonanMetaFileTypeFactory : FileTypeFactory() { + + override fun createFileTypes(consumer: FileTypeConsumer) = consumer.consume(KonanMetaFileType, KonanMetaFileType.defaultExtension) +} diff --git a/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataDecompilerBase.kt b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataDecompilerBase.kt new file mode 100644 index 00000000000..6d2c89f6abd --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataDecompilerBase.kt @@ -0,0 +1,121 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.analyser.index + +import com.intellij.openapi.fileTypes.FileType +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.psi.PsiManager +import com.intellij.psi.compiled.ClassFileDecompilers +import org.jetbrains.konan.KonanDecompiledFile +import org.jetbrains.kotlin.descriptors.DeclarationDescriptor +import org.jetbrains.kotlin.idea.decompiler.KotlinDecompiledFileViewProvider +import org.jetbrains.kotlin.idea.decompiler.common.createIncompatibleAbiVersionDecompiledText +import org.jetbrains.kotlin.idea.decompiler.textBuilder.DecompiledText +import org.jetbrains.kotlin.idea.decompiler.textBuilder.buildDecompiledText +import org.jetbrains.kotlin.idea.decompiler.textBuilder.defaultDecompilerRendererOptions +import org.jetbrains.kotlin.metadata.ProtoBuf +import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion +import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl +import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.renderer.DescriptorRenderer +import org.jetbrains.kotlin.resolve.TargetPlatform +import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol +import org.jetbrains.kotlin.serialization.deserialization.ClassDeserializer +import org.jetbrains.kotlin.serialization.deserialization.FlexibleTypeDeserializer +import org.jetbrains.kotlin.serialization.deserialization.getClassId +import org.jetbrains.kotlin.utils.addIfNotNull +import java.io.IOException + +//todo: Fix in Kotlin plugin +abstract class KonanMetadataDecompilerBase( + private val fileType: FileType, + private val targetPlatform: TargetPlatform, + private val serializerProtocol: SerializerExtensionProtocol, + private val flexibleTypeDeserializer: FlexibleTypeDeserializer, + private val expectedBinaryVersion: V, + private val invalidBinaryVersion: V, + stubVersion: Int +) : ClassFileDecompilers.Full() { + + private val stubBuilder = KonanMetadataStubBuilder(stubVersion, fileType, serializerProtocol, ::readFileSafely) + + private val renderer = DescriptorRenderer.withOptions { defaultDecompilerRendererOptions() } + + protected abstract fun doReadFile(file: VirtualFile): FileWithMetadata? + + override fun accepts(file: VirtualFile) = file.fileType == fileType + + override fun getStubBuilder() = stubBuilder + + override fun createFileViewProvider(file: VirtualFile, manager: PsiManager, physical: Boolean) = + KotlinDecompiledFileViewProvider(manager, file, physical) { provider -> KonanDecompiledFile(provider, ::buildDecompiledText) } + + private fun readFileSafely(file: VirtualFile): FileWithMetadata? { + if (!file.isValid) return null + + return try { + doReadFile(file) + } catch (e: IOException) { + // This is needed because sometimes we're given VirtualFile instances that point to non-existent .jar entries. + // Such files are valid (isValid() returns true), but an attempt to read their contents results in a FileNotFoundException. + // Note that although calling "refresh()" instead of catching an exception would seem more correct here, + // it's not always allowed and also is likely to degrade performance + null + } + } + + private fun buildDecompiledText(virtualFile: VirtualFile): DecompiledText { + assert(virtualFile.fileType == fileType) { "Unexpected file type ${virtualFile.fileType}" } + + val file = readFileSafely(virtualFile) + + return when (file) { + is FileWithMetadata.Incompatible -> createIncompatibleAbiVersionDecompiledText(expectedBinaryVersion, file.version) + is FileWithMetadata.Compatible -> decompiledText(file, targetPlatform, serializerProtocol, flexibleTypeDeserializer, renderer) + null -> createIncompatibleAbiVersionDecompiledText(expectedBinaryVersion, invalidBinaryVersion) + } + } +} + +sealed class FileWithMetadata { + class Incompatible(val version: BinaryVersion) : FileWithMetadata() + + open class Compatible( + val proto: KonanProtoBuf.LinkDataPackageFragment, + serializerProtocol: SerializerExtensionProtocol + ) : FileWithMetadata() { + val nameResolver = NameResolverImpl(proto.stringTable, proto.nameTable) + val packageFqName = FqName(nameResolver.getPackageFqName(proto.`package`.getExtension(serializerProtocol.packageFqName))) + + open val classesToDecompile: List = + proto.classes.classesList.filter { proto -> + val classId = nameResolver.getClassId(proto.fqName) + !classId.isNestedClass && classId !in ClassDeserializer.BLACK_LIST + } + } +} + +//todo: this function is extracted for KonanMetadataStubBuilder, that's the difference from Big Kotlin. +fun decompiledText( + file: FileWithMetadata.Compatible, targetPlatform: TargetPlatform, + serializerProtocol: SerializerExtensionProtocol, + flexibleTypeDeserializer: FlexibleTypeDeserializer, + renderer: DescriptorRenderer +): DecompiledText { + val packageFqName = file.packageFqName + val resolver = KonanMetadataDeserializerForDecompiler( + packageFqName, file.proto, file.nameResolver, + targetPlatform, serializerProtocol, flexibleTypeDeserializer + ) + val declarations = arrayListOf() + declarations.addAll(resolver.resolveDeclarationsInFacade(packageFqName)) + for (classProto in file.classesToDecompile) { + val classId = file.nameResolver.getClassId(classProto.fqName) + declarations.addIfNotNull(resolver.resolveTopLevelClass(classId)) + } + return buildDecompiledText(packageFqName, declarations, renderer) +} diff --git a/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataDeserializerForDecompiler.kt b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataDeserializerForDecompiler.kt new file mode 100644 index 00000000000..53d37729286 --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataDeserializerForDecompiler.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + + +package org.jetbrains.konan.analyser.index + +import com.intellij.openapi.diagnostic.Logger +import org.jetbrains.konan.createLoggingErrorReporter +import org.jetbrains.kotlin.builtins.DefaultBuiltIns +import org.jetbrains.kotlin.builtins.KotlinBuiltIns +import org.jetbrains.kotlin.descriptors.DeclarationDescriptor +import org.jetbrains.kotlin.descriptors.NotFoundClasses +import org.jetbrains.kotlin.idea.decompiler.textBuilder.DeserializerForDecompilerBase +import org.jetbrains.kotlin.idea.decompiler.textBuilder.ResolveEverythingToKotlinAnyLocalClassifierResolver +import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.metadata.deserialization.NameResolver +import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.resolve.TargetPlatform +import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol +import org.jetbrains.kotlin.serialization.deserialization.* +import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope + +//todo: Fix in Kotlin plugin +class KonanMetadataDeserializerForDecompiler( + packageFqName: FqName, + private val proto: KonanProtoBuf.LinkDataPackageFragment, + private val nameResolver: NameResolver, + override val targetPlatform: TargetPlatform, + serializerProtocol: SerializerExtensionProtocol, + flexibleTypeDeserializer: FlexibleTypeDeserializer +) : DeserializerForDecompilerBase(packageFqName) { + override val builtIns: KotlinBuiltIns get() = DefaultBuiltIns.Instance + + override val deserializationComponents: DeserializationComponents + + init { + val notFoundClasses = NotFoundClasses(storageManager, moduleDescriptor) + + deserializationComponents = DeserializationComponents( + storageManager, moduleDescriptor, DeserializationConfiguration.Default, KonanProtoBasedClassDataFinder(proto, nameResolver), + AnnotationAndConstantLoaderImpl(moduleDescriptor, notFoundClasses, serializerProtocol), packageFragmentProvider, + ResolveEverythingToKotlinAnyLocalClassifierResolver(builtIns), createLoggingErrorReporter(LOG), + LookupTracker.DO_NOTHING, flexibleTypeDeserializer, emptyList(), notFoundClasses, ContractDeserializer.DEFAULT, + extensionRegistryLite = serializerProtocol.extensionRegistry + ) + } + + override fun resolveDeclarationsInFacade(facadeFqName: FqName): List { + assert(facadeFqName == directoryPackageFqName) { + "Was called for $facadeFqName; only members of $directoryPackageFqName package are expected." + } + + val membersScope = DeserializedPackageMemberScope( + createDummyPackageFragment(facadeFqName), + proto.`package`, + nameResolver, + KonanMetadataVersion.DEFAULT_INSTANCE, + containerSource = null, + components = deserializationComponents + ) { emptyList() } + + return membersScope.getContributedDescriptors().toList() + } + + companion object { + private val LOG = Logger.getInstance(KonanMetadataDeserializerForDecompiler::class.java) + } +} + diff --git a/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataStubBuilder.kt b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataStubBuilder.kt new file mode 100644 index 00000000000..89fcbc67335 --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanMetadataStubBuilder.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.analyser.index + +import com.intellij.openapi.fileTypes.FileType +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.psi.compiled.ClsStubBuilder +import com.intellij.psi.impl.compiled.ClassFileStubBuilder +import com.intellij.psi.stubs.PsiFileStub +import com.intellij.util.indexing.FileContent +import org.jetbrains.konan.createFileStub +import org.jetbrains.kotlin.idea.decompiler.stubBuilder.createIncompatibleAbiVersionFileStub +import org.jetbrains.kotlin.idea.decompiler.textBuilder.defaultDecompilerRendererOptions +import org.jetbrains.kotlin.renderer.DescriptorRenderer +import org.jetbrains.kotlin.resolve.konan.platform.KonanPlatform +import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol +import org.jetbrains.kotlin.serialization.konan.NullFlexibleTypeDeserializer + +//todo: Fix in Kotlin plugin +open class KonanMetadataStubBuilder( + private val version: Int, + private val fileType: FileType, + private val serializerProtocol: SerializerExtensionProtocol, + private val readFile: (VirtualFile) -> FileWithMetadata? +) : ClsStubBuilder() { + + override fun getStubVersion() = ClassFileStubBuilder.STUB_VERSION + version + + override fun buildFileStub(content: FileContent): PsiFileStub<*>? { + val virtualFile = content.file + assert(virtualFile.fileType == fileType) { "Unexpected file type ${virtualFile.fileType}" } + + val file = readFile(virtualFile) ?: return null + + return when (file) { + is FileWithMetadata.Incompatible -> createIncompatibleAbiVersionFileStub() + is FileWithMetadata.Compatible -> { //todo: this part is implemented in our own way + val renderer = DescriptorRenderer.withOptions { defaultDecompilerRendererOptions() } + val ktFileText = decompiledText(file, KonanPlatform, serializerProtocol, NullFlexibleTypeDeserializer, renderer) + createFileStub(content.project, ktFileText.text) + } + } + } +} diff --git a/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanProtoBasedClassDataFinder.kt b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanProtoBasedClassDataFinder.kt new file mode 100644 index 00000000000..e3d71c390df --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/analyser/index/KonanProtoBasedClassDataFinder.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.analyser.index + +import org.jetbrains.kotlin.descriptors.SourceElement +import org.jetbrains.kotlin.metadata.deserialization.NameResolver +import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf +import org.jetbrains.kotlin.name.ClassId +import org.jetbrains.kotlin.serialization.deserialization.ClassData +import org.jetbrains.kotlin.serialization.deserialization.ClassDataFinder +import org.jetbrains.kotlin.serialization.deserialization.getClassId + +//todo: Fix in Kotlin plugin +class KonanProtoBasedClassDataFinder( + proto: KonanProtoBuf.LinkDataPackageFragment, + private val nameResolver: NameResolver, + private val classSource: (ClassId) -> SourceElement = { SourceElement.NO_SOURCE } +) : ClassDataFinder { + private val classIdToProto = + proto.classes.classesList.associateBy { klass -> + nameResolver.getClassId(klass.fqName) + } + + internal val allClassIds: Collection get() = classIdToProto.keys + + override fun findClassData(classId: ClassId): ClassData? { + val classProto = classIdToProto[classId] ?: return null + return ClassData(nameResolver, classProto, KonanMetadataVersion.DEFAULT_INSTANCE, classSource(classId)) + } +} diff --git a/idea/idea-native/src/org/jetbrains/konan/settings/KonanArtifact.kt b/idea/idea-native/src/org/jetbrains/konan/settings/KonanArtifact.kt new file mode 100644 index 00000000000..df74c32d262 --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/settings/KonanArtifact.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.settings + +import org.jetbrains.kotlin.konan.target.CompilerOutputKind +import org.jetbrains.kotlin.konan.target.CompilerOutputKind.* +import org.jetbrains.kotlin.konan.target.KonanTarget +import java.nio.file.Path + +data class KonanArtifact( + val targetName: String, + val moduleName: String, + val type: CompilerOutputKind, + val target: KonanTarget?, + val libraryDependencies: List, + val sources: List, + val output: Path +) + +val CompilerOutputKind.isLibrary: Boolean get() = this == LIBRARY || this == DYNAMIC || this == STATIC || this == FRAMEWORK +val CompilerOutputKind.isExecutable: Boolean get() = this == PROGRAM +val CompilerOutputKind.isTest: Boolean get() = false //TODO diff --git a/idea/idea-native/src/org/jetbrains/konan/settings/KonanModelProvider.java b/idea/idea-native/src/org/jetbrains/konan/settings/KonanModelProvider.java new file mode 100644 index 00000000000..912b8cabe45 --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/settings/KonanModelProvider.java @@ -0,0 +1,31 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.settings; + +import com.intellij.openapi.extensions.ExtensionPointName; +import com.intellij.openapi.project.Project; +import com.intellij.util.messages.Topic; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.nio.file.Path; +import java.util.Collection; + +@ApiStatus.Experimental +public interface KonanModelProvider { + Topic RELOAD_TOPIC = new Topic<>("Kotlin/Native Project Model Updater", Runnable.class); + + ExtensionPointName EP_NAME = ExtensionPointName.create("org.jetbrains.kotlin.native.konanModelProvider"); + + @NotNull + Collection getArtifacts(@NotNull Project project); + + @Nullable + Path getKonanHome(@NotNull Project project); + + boolean reloadLibraries(@NotNull Project project, @NotNull Collection libraryPaths); +} diff --git a/idea/idea-native/src/org/jetbrains/konan/settings/KonanPaths.kt b/idea/idea-native/src/org/jetbrains/konan/settings/KonanPaths.kt new file mode 100644 index 00000000000..780720c2bdb --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/settings/KonanPaths.kt @@ -0,0 +1,53 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.settings + +import com.intellij.openapi.components.ProjectComponent +import com.intellij.openapi.project.Project +import com.intellij.util.io.exists +import com.intellij.util.io.isDirectory +import org.jetbrains.konan.KotlinNativeToolchain +import org.jetbrains.kotlin.konan.library.* +import org.jetbrains.kotlin.konan.target.HostManager +import org.jetbrains.kotlin.konan.target.KonanTarget +import java.nio.file.Files +import java.nio.file.Path +import java.util.stream.Collectors + +open class KonanPaths(protected val project: Project) : ProjectComponent { + companion object { + fun getInstance(project: Project): KonanPaths = project.getComponent(KonanPaths::class.java) + + fun bundledKonanDist(): Path = KotlinNativeToolchain.BUNDLED.baseDir + } + + override fun getComponentName() = "Kotlin/Native Compiler Paths" + + fun konanStdlib(): Path? = konanDist()?.resolve(konanCommonLibraryPath(KONAN_STDLIB_NAME)) + + open fun konanDist(): Path? { + for (provider in KonanModelProvider.EP_NAME.extensions) { + return provider.getKonanHome(project) + } + return bundledKonanDist() + } + + open fun libraryPaths(): Set = emptySet() + + fun konanPlatformLibraries(): List { + + val resolvedTargetName = HostManager.resolveAlias(target().name) + val klibPath = + konanDist()?.resolve(konanSpecificPlatformLibrariesPath(resolvedTargetName))?.takeIf { it.exists() } ?: return emptyList() + + return Files.walk(klibPath, 1).filter { + it.isDirectory() && it.fileName.toString() != KONAN_STDLIB_NAME && it != klibPath + }.collect(Collectors.toList()) + } + + //todo: this is wrong, we are not allowing multiple targets in project + open fun target(): KonanTarget = HostManager.host +} diff --git a/idea/idea-native/src/org/jetbrains/konan/settings/KonanProjectComponent.kt b/idea/idea-native/src/org/jetbrains/konan/settings/KonanProjectComponent.kt new file mode 100644 index 00000000000..80eae4f2db3 --- /dev/null +++ b/idea/idea-native/src/org/jetbrains/konan/settings/KonanProjectComponent.kt @@ -0,0 +1,104 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.konan.settings + +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.application.runWriteAction +import com.intellij.openapi.components.ProjectComponent +import com.intellij.openapi.module.ModuleManager +import com.intellij.openapi.project.Project +import com.intellij.openapi.roots.ModuleRootManager +import com.intellij.openapi.roots.OrderRootType +import com.intellij.openapi.roots.libraries.LibraryTable +import com.intellij.openapi.vfs.* +import org.jetbrains.konan.KotlinNativeToolchain +import org.jetbrains.konan.settings.KonanModelProvider.RELOAD_TOPIC +import org.jetbrains.kotlin.utils.addIfNotNull +import java.nio.file.Path + +abstract class KonanProjectComponent(val project: Project) : ProjectComponent { + companion object { + fun getInstance(project: Project): KonanProjectComponent = project.getComponent(KonanProjectComponent::class.java) + } + + protected var libraryPaths: Set = emptySet() + + override fun getComponentName(): String = "Kotlin/Native Project Component" + + override fun projectOpened() { + VirtualFileManager.getInstance().addVirtualFileListener(object : VirtualFileListener { + override fun fileCreated(event: VirtualFileEvent) { + val filePath = event.file.path + if (libraryPaths.contains(filePath)) reloadLibraries() + } + }, project) + + val connection = project.messageBus.connect(project) + connection.subscribe(RELOAD_TOPIC, Runnable { + ApplicationManager.getApplication().invokeLater { + KotlinNativeToolchain.BUNDLED.ensureExists(project) + reloadLibraries() + } + }) + + if (looksLikeKotlinNativeProject()) { + reloadLibraries() + } + } + + open fun reloadLibraries(): Unit = synchronized(this) { + val libraryPaths: Set = collectLibraryPaths() + this.libraryPaths = libraryPaths.asSequence().map(Path::toString).toSet() + +// for (konanModelProvider in KonanModelProvider.EP_NAME.extensions) { +// if (konanModelProvider.reloadLibraries(project, libraryPaths)) return +// } + + val module = ModuleManager.getInstance(project).modules.firstOrNull() ?: return + val modifiableModel = ModuleRootManager.getInstance(module).modifiableModel + val libraryTable: LibraryTable = modifiableModel.moduleLibraryTable + libraryTable.libraries.forEach { libraryTable.removeLibrary(it) } + + val localFileSystem = VirtualFileManager.getInstance().getFileSystem(StandardFileSystems.FILE_PROTOCOL) + val jarFileSystem = VirtualFileManager.getInstance().getFileSystem(StandardFileSystems.JAR_PROTOCOL) + + for (path in libraryPaths) { + + fun createLibrary(vfs: VirtualFileSystem, rootPath: String) { + val root = vfs.refreshAndFindFileByPath(rootPath) ?: return + + val library = libraryTable.createLibrary(path.fileName.toString()) + val libraryModifiableModel = library.modifiableModel + + libraryModifiableModel.addRoot(root, OrderRootType.CLASSES) + + runWriteAction { + libraryModifiableModel.commit() + } + } + + val libraryRootPath = path.toString() + val libraryRoot: VirtualFile = localFileSystem.refreshAndFindFileByPath(libraryRootPath) ?: continue + + // use JAR FS if this is file (*.klib), otherwise - local FS + createLibrary(if (libraryRoot.isDirectory) localFileSystem else jarFileSystem, libraryRootPath) + } + + runWriteAction { + modifiableModel.commit() + } + } + + protected open fun collectLibraryPaths(): MutableSet { + val konanPaths = KonanPaths.getInstance(project) + return mutableSetOf().apply { + addIfNotNull(konanPaths.konanStdlib()) + addAll(konanPaths.konanPlatformLibraries()) + } + } + + abstract fun looksLikeKotlinNativeProject(): Boolean +} \ No newline at end of file diff --git a/idea/src/META-INF/gradle.xml b/idea/src/META-INF/gradle.xml index 561899e1ec7..5201b4b795e 100644 --- a/idea/src/META-INF/gradle.xml +++ b/idea/src/META-INF/gradle.xml @@ -10,4 +10,31 @@ + + + + + org.jetbrains.konan.gradle.internal.KotlinNativeIdeInitializer + + + + + + org.jetbrains.konan.settings.KonanProjectComponent + org.jetbrains.konan.gradle.GradleKonanProjectComponent + + + + + + + + + + + + + + + diff --git a/idea/src/META-INF/native.xml b/idea/src/META-INF/native.xml new file mode 100644 index 00000000000..5ce64f5a7ef --- /dev/null +++ b/idea/src/META-INF/native.xml @@ -0,0 +1,37 @@ + + + + org.jetbrains.konan.KonanApplicationComponent + + + org.jetbrains.konan.analyser.index.KonanDescriptorManager + + + + + + org.jetbrains.konan.settings.KonanPaths + + + + + + + + + + + + + + + + + + + + + + + diff --git a/idea/src/META-INF/plugin.xml b/idea/src/META-INF/plugin.xml index 2ec09d82368..420d1fce527 100644 --- a/idea/src/META-INF/plugin.xml +++ b/idea/src/META-INF/plugin.xml @@ -3097,6 +3097,7 @@ The Kotlin plugin provides language support in IntelliJ IDEA and Android Studio. + diff --git a/idea/src/META-INF/plugin.xml.183 b/idea/src/META-INF/plugin.xml.183 index 238f057762f..30aa9929c94 100644 --- a/idea/src/META-INF/plugin.xml.183 +++ b/idea/src/META-INF/plugin.xml.183 @@ -3097,6 +3097,7 @@ The Kotlin plugin provides language support in IntelliJ IDEA and Android Studio. + diff --git a/idea/src/META-INF/plugin.xml.as32 b/idea/src/META-INF/plugin.xml.as32 index 8ad38b86f93..92be838a3d4 100644 --- a/idea/src/META-INF/plugin.xml.as32 +++ b/idea/src/META-INF/plugin.xml.as32 @@ -3095,6 +3095,7 @@ The Kotlin plugin provides language support in IntelliJ IDEA and Android Studio. + diff --git a/idea/src/META-INF/plugin.xml.as33 b/idea/src/META-INF/plugin.xml.as33 index a723539d8db..ace6d400bb9 100644 --- a/idea/src/META-INF/plugin.xml.as33 +++ b/idea/src/META-INF/plugin.xml.as33 @@ -3097,6 +3097,7 @@ The Kotlin plugin provides language support in IntelliJ IDEA and Android Studio. + diff --git a/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/KonanDeserializedModuleDescriptorFactory.kt b/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/KonanDeserializedModuleDescriptorFactory.kt index 6c006c1ab11..1f9e02036ef 100644 --- a/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/KonanDeserializedModuleDescriptorFactory.kt +++ b/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/KonanDeserializedModuleDescriptorFactory.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.serialization.konan import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.config.LanguageVersionSettings +import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.descriptors.konan.KonanModuleDescriptorFactory import org.jetbrains.kotlin.konan.library.KonanLibrary @@ -24,7 +25,8 @@ interface KonanDeserializedModuleDescriptorFactory { languageVersionSettings: LanguageVersionSettings, storageManager: StorageManager, builtIns: KotlinBuiltIns, - packageAccessedHandler: PackageAccessedHandler? = null + packageAccessedHandler: PackageAccessedHandler? = null, + customCapabilities: Map, Any?> = emptyMap() ): ModuleDescriptorImpl /** @@ -35,6 +37,7 @@ interface KonanDeserializedModuleDescriptorFactory { library: KonanLibrary, languageVersionSettings: LanguageVersionSettings, storageManager: StorageManager, - packageAccessedHandler: PackageAccessedHandler? = null + packageAccessedHandler: PackageAccessedHandler? = null, + customCapabilities: Map, Any?> = emptyMap() ): ModuleDescriptorImpl } diff --git a/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/KonanResolvedModuleDescriptorsFactory.kt b/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/KonanResolvedModuleDescriptorsFactory.kt index 0bfd417c20a..0bb474ffe87 100644 --- a/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/KonanResolvedModuleDescriptorsFactory.kt +++ b/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/KonanResolvedModuleDescriptorsFactory.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.serialization.konan import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.config.LanguageVersionSettings +import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.konan.library.KonanLibrary import org.jetbrains.kotlin.konan.library.resolver.KonanLibraryResolveResult @@ -33,7 +34,8 @@ interface KonanResolvedModuleDescriptorsFactory { storageManager: StorageManager, builtIns: KotlinBuiltIns?, languageVersionSettings: LanguageVersionSettings, - customAction: ((KonanLibrary, ModuleDescriptorImpl) -> Unit)? = null + customAction: ((KonanLibrary, ModuleDescriptorImpl) -> Unit)? = null, + customCapabilitiesGenerator: ((KonanLibrary) -> Map, Any?>)? = null ): KonanResolvedModuleDescriptors } diff --git a/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/impl/KonanDeserializedModuleDescriptorFactoryImpl.kt b/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/impl/KonanDeserializedModuleDescriptorFactoryImpl.kt index f6aed45eb64..25705a0bd99 100644 --- a/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/impl/KonanDeserializedModuleDescriptorFactoryImpl.kt +++ b/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/impl/KonanDeserializedModuleDescriptorFactoryImpl.kt @@ -33,22 +33,32 @@ internal class KonanDeserializedModuleDescriptorFactoryImpl( languageVersionSettings: LanguageVersionSettings, storageManager: StorageManager, builtIns: KotlinBuiltIns, - packageAccessedHandler: PackageAccessedHandler? - ) = createDescriptorOptionalBuiltIns(library, languageVersionSettings, storageManager, builtIns, packageAccessedHandler) + packageAccessedHandler: PackageAccessedHandler?, + customCapabilities: Map, Any?> + ) = createDescriptorOptionalBuiltIns( + library, + languageVersionSettings, + storageManager, + builtIns, + packageAccessedHandler, + customCapabilities + ) override fun createDescriptorAndNewBuiltIns( library: KonanLibrary, languageVersionSettings: LanguageVersionSettings, storageManager: StorageManager, - packageAccessedHandler: PackageAccessedHandler? - ) = createDescriptorOptionalBuiltIns(library, languageVersionSettings, storageManager, null, packageAccessedHandler) + packageAccessedHandler: PackageAccessedHandler?, + customCapabilities: Map, Any?> + ) = createDescriptorOptionalBuiltIns(library, languageVersionSettings, storageManager, null, packageAccessedHandler, customCapabilities) private fun createDescriptorOptionalBuiltIns( library: KonanLibrary, languageVersionSettings: LanguageVersionSettings, storageManager: StorageManager, builtIns: KotlinBuiltIns?, - packageAccessedHandler: PackageAccessedHandler? + packageAccessedHandler: PackageAccessedHandler?, + customCapabilities: Map, Any?> ): ModuleDescriptorImpl { val libraryProto = parseModuleHeader(library.moduleHeaderData) @@ -57,9 +67,9 @@ internal class KonanDeserializedModuleDescriptorFactoryImpl( val moduleOrigin = DeserializedKonanModuleOrigin(library) val moduleDescriptor = if (builtIns != null) - descriptorFactory.createDescriptor(moduleName, storageManager, builtIns, moduleOrigin) + descriptorFactory.createDescriptor(moduleName, storageManager, builtIns, moduleOrigin, customCapabilities) else - descriptorFactory.createDescriptorAndNewBuiltIns(moduleName, storageManager, moduleOrigin) + descriptorFactory.createDescriptorAndNewBuiltIns(moduleName, storageManager, moduleOrigin, customCapabilities) val deserializationConfiguration = CompilerDeserializationConfiguration(languageVersionSettings) diff --git a/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/impl/KonanResolvedModuleDescriptorsFactoryImpl.kt b/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/impl/KonanResolvedModuleDescriptorsFactoryImpl.kt index aee43936170..9aacb35c632 100644 --- a/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/impl/KonanResolvedModuleDescriptorsFactoryImpl.kt +++ b/konan/serializer/src/org/jetbrains/kotlin/serialization/konan/impl/KonanResolvedModuleDescriptorsFactoryImpl.kt @@ -41,7 +41,8 @@ class KonanResolvedModuleDescriptorsFactoryImpl( storageManager: StorageManager, builtIns: KotlinBuiltIns?, languageVersionSettings: LanguageVersionSettings, - customAction: ((KonanLibrary, ModuleDescriptorImpl) -> Unit)? + customAction: ((KonanLibrary, ModuleDescriptorImpl) -> Unit)?, + customCapabilitiesGenerator: ((KonanLibrary) -> Map, Any?>)? ): KonanResolvedModuleDescriptors { val moduleDescriptors = mutableListOf() @@ -52,9 +53,11 @@ class KonanResolvedModuleDescriptorsFactoryImpl( resolvedLibraries.forEach { library, packageAccessedHandler -> profile("Loading ${library.libraryName}") { + val customCapabilities = customCapabilitiesGenerator?.invoke(library) ?: emptyMap() + // MutableModuleContext needs ModuleDescriptorImpl, rather than ModuleDescriptor. val moduleDescriptor = createDescriptorOptionalBuiltsIns( - library, languageVersionSettings, storageManager, builtIns, packageAccessedHandler + library, languageVersionSettings, storageManager, builtIns, packageAccessedHandler, customCapabilities ) builtIns = moduleDescriptor.builtIns moduleDescriptors.add(moduleDescriptor) @@ -120,11 +123,25 @@ class KonanResolvedModuleDescriptorsFactoryImpl( languageVersionSettings: LanguageVersionSettings, storageManager: StorageManager, builtIns: KotlinBuiltIns?, - packageAccessedHandler: PackageAccessedHandler? + packageAccessedHandler: PackageAccessedHandler?, + customCapabilities: Map, Any?> ) = if (builtIns != null) - moduleDescriptorFactory.createDescriptor(library, languageVersionSettings, storageManager, builtIns, packageAccessedHandler) + moduleDescriptorFactory.createDescriptor( + library, + languageVersionSettings, + storageManager, + builtIns, + packageAccessedHandler, + customCapabilities + ) else - moduleDescriptorFactory.createDescriptorAndNewBuiltIns(library, languageVersionSettings, storageManager, packageAccessedHandler) + moduleDescriptorFactory.createDescriptorAndNewBuiltIns( + library, + languageVersionSettings, + storageManager, + packageAccessedHandler, + customCapabilities + ) } /** diff --git a/konan/utils/src/org/jetbrains/kotlin/konan/library/KonanLibraryConstants.kt b/konan/utils/src/org/jetbrains/kotlin/konan/library/KonanLibraryConstants.kt new file mode 100644 index 00000000000..b149a1712b4 --- /dev/null +++ b/konan/utils/src/org/jetbrains/kotlin/konan/library/KonanLibraryConstants.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.library + +import java.nio.file.Path +import java.nio.file.Paths + +const val KLIB_FILE_EXTENSION = "klib" +const val KLIB_FILE_EXTENSION_WITH_DOT = ".$KLIB_FILE_EXTENSION" + +const val KONAN_STDLIB_NAME = "stdlib" + +val KONAN_COMMON_LIBS_PATH: Path + get() = Paths.get("klib", "common") + +val KONAN_ALL_PLATFORM_LIBS_PATH: Path + get() = Paths.get("klib", "platform") + +fun konanCommonLibraryPath(libraryName: String): Path = KONAN_COMMON_LIBS_PATH.resolve(libraryName) + +fun konanSpecificPlatformLibrariesPath(platform: String): Path = KONAN_ALL_PLATFORM_LIBS_PATH.resolve(platform) + +fun konanPlatformLibraryPath(platform: String, libraryName: String): Path = konanSpecificPlatformLibrariesPath(platform).resolve(libraryName) diff --git a/konan/utils/src/org/jetbrains/kotlin/konan/library/SearchPathResolver.kt b/konan/utils/src/org/jetbrains/kotlin/konan/library/SearchPathResolver.kt index aa9eee12250..7e7ef02cec7 100644 --- a/konan/utils/src/org/jetbrains/kotlin/konan/library/SearchPathResolver.kt +++ b/konan/utils/src/org/jetbrains/kotlin/konan/library/SearchPathResolver.kt @@ -8,17 +8,12 @@ package org.jetbrains.kotlin.konan.library import org.jetbrains.kotlin.konan.file.File import org.jetbrains.kotlin.konan.target.KonanTarget -const val KLIB_FILE_EXTENSION = "klib" -const val KLIB_FILE_EXTENSION_WITH_DOT = ".$KLIB_FILE_EXTENSION" - -const val KONAN_STDLIB_NAME = "stdlib" - interface SearchPathResolver { val searchRoots: List fun resolve(givenPath: String): File fun defaultLinks(noStdLib: Boolean, noDefaultLibs: Boolean): List } -interface SearchPathResolverWithTarget: SearchPathResolver { +interface SearchPathResolverWithTarget : SearchPathResolver { val target: KonanTarget } diff --git a/konan/utils/src/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt b/konan/utils/src/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt new file mode 100644 index 00000000000..60a3b6b3312 --- /dev/null +++ b/konan/utils/src/org/jetbrains/kotlin/konan/util/DependencyDownloader.kt @@ -0,0 +1,216 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.util + +import java.io.* +import java.net.HttpURLConnection +import java.net.URL +import java.net.URLConnection +import java.nio.file.Files +import java.nio.file.StandardCopyOption +import java.util.concurrent.* + +internal typealias ProgressCallback = (url: String, currentBytes: Long, totalBytes: Long) -> Unit + +internal class DependencyDownloader( + var maxAttempts: Int = DEFAULT_MAX_ATTEMPTS, + var attemptIntervalMs: Long = DEFAULT_ATTEMPT_INTERVAL_MS, + customProgressCallback: ProgressCallback? = null +) { + + private val progressCallback = customProgressCallback ?: { url, currentBytes, totalBytes -> + print("\rDownloading dependency: $url (${currentBytes.humanReadable}/${totalBytes.humanReadable}). ") + } + + val executor = ExecutorCompletionService(Executors.newSingleThreadExecutor { r -> + val thread = Thread(r) + thread.name = "konan-dependency-downloader" + thread.isDaemon = true + + thread + }) + + enum class ReplacingMode { + /** Re-download the file and replace the existing one. */ + REPLACE, + /** Throw FileAlreadyExistsException */ + THROW, + /** Don't download the file and return the existing one*/ + RETURN_EXISTING + } + + class DownloadingProgress(@Volatile var currentBytes: Long) { + fun update(readBytes: Int) { + currentBytes += readBytes + } + } + + private fun HttpURLConnection.checkHTTPResponse(expected: Int, originalUrl: URL = url) { + if (responseCode != expected) { + throw DependencyDownloaderHTTPResponseException(originalUrl, responseCode) + } + } + + private fun HttpURLConnection.checkHTTPResponse(originalUrl: URL, predicate: (Int) -> Boolean) { + if (!predicate(responseCode)) { + throw DependencyDownloaderHTTPResponseException(originalUrl, responseCode) + } + } + + private fun doDownload( + originalUrl: URL, + connection: URLConnection, + tmpFile: File, + currentBytes: Long, + totalBytes: Long, + append: Boolean + ) { + val progress = DownloadingProgress(currentBytes) + + // TODO: Implement multi-thread downloading. + executor.submit { + connection.getInputStream().use { from -> + FileOutputStream(tmpFile, append).use { to -> + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + var read = from.read(buffer) + while (read != -1) { + if (Thread.interrupted()) { + throw InterruptedException() + } + to.write(buffer, 0, read) + progress.update(read) + read = from.read(buffer) + } + if (progress.currentBytes != totalBytes) { + throw EOFException("The stream closed before end of downloading.") + } + } + } + } + + var result: Future? + do { + progressCallback(originalUrl.toString(), progress.currentBytes, totalBytes) + result = executor.poll(1, TimeUnit.SECONDS) + } while (result == null) + progressCallback(originalUrl.toString(), progress.currentBytes, totalBytes) + + try { + result.get() + } catch (e: ExecutionException) { + throw e.cause ?: e + } + } + + private fun resumeDownload(originalUrl: URL, originalConnection: HttpURLConnection, tmpFile: File) { + originalConnection.connect() + val totalBytes = originalConnection.contentLengthLong + val currentBytes = tmpFile.length() + if (currentBytes >= totalBytes || originalConnection.getHeaderField("Accept-Ranges") != "bytes") { + // The temporary file is bigger then expected or the server doesn't support resuming downloading. + // Download the file from scratch. + doDownload(originalUrl, originalConnection, tmpFile, 0, totalBytes, false) + } else { + originalConnection.disconnect() + val rangeConnection = originalUrl.openConnection() as HttpURLConnection + rangeConnection.setRequestProperty("range", "bytes=$currentBytes-") + rangeConnection.connect() + rangeConnection.checkHTTPResponse(originalUrl) { + it == HttpURLConnection.HTTP_PARTIAL || it == HttpURLConnection.HTTP_OK + } + doDownload(originalUrl, rangeConnection, tmpFile, currentBytes, totalBytes, true) + } + } + + /** Performs an attempt to download a specified file into the specified location */ + private fun tryDownload(url: URL, tmpFile: File) { + val connection = url.openConnection() + + (connection as? HttpURLConnection)?.checkHTTPResponse(HttpURLConnection.HTTP_OK, url) + + if (connection is HttpURLConnection && tmpFile.exists()) { + resumeDownload(url, connection, tmpFile) + } else { + connection.connect() + val totalBytes = connection.contentLengthLong + doDownload(url, connection, tmpFile, 0, totalBytes, false) + } + } + + /** Downloads a file from [source] url to [destination]. Returns [destination]. */ + fun download( + source: URL, + destination: File, + replace: ReplacingMode = ReplacingMode.RETURN_EXISTING + ): File { + + if (destination.exists()) { + when (replace) { + ReplacingMode.RETURN_EXISTING -> return destination + ReplacingMode.THROW -> throw FileAlreadyExistsException(destination) + ReplacingMode.REPLACE -> Unit // Just continue with downloading. + } + } + val tmpFile = File("${destination.canonicalPath}.$TMP_SUFFIX") + + check(!tmpFile.isDirectory) { + "A temporary file is a directory: ${tmpFile.canonicalPath}. Remove it and try again." + } + check(!destination.isDirectory) { + "The destination file is a directory: ${destination.canonicalPath}. Remove it and try again." + } + + var attempt = 1 + var waitTime = 0L + while (true) { + try { + tryDownload(source, tmpFile) + break + } catch (e: DependencyDownloaderHTTPResponseException) { + throw e + } catch (e: IOException) { + if (attempt >= maxAttempts) { + throw e + } + attempt++ + waitTime += attemptIntervalMs + println( + "Cannot download a dependency: $e\n" + + "Waiting ${waitTime.toDouble() / 1000} sec and trying again (attempt: $attempt/$maxAttempts)." + ) + // TODO: Wait better + Thread.sleep(waitTime) + } + } + + Files.move(tmpFile.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING) + println("Done.") + return destination + } + + private val Long.humanReadable: String + get() { + if (this < 0) { + return "-" + } + if (this < 1024) { + return "$this bytes" + } + val exp = (Math.log(this.toDouble()) / Math.log(1024.0)).toInt() + val prefix = "kMGTPE"[exp - 1] + return "%.1f %siB".format(this / Math.pow(1024.0, exp.toDouble()), prefix) + } + + companion object { + const val DEFAULT_MAX_ATTEMPTS = 10 + const val DEFAULT_ATTEMPT_INTERVAL_MS = 3000L + + const val TMP_SUFFIX = "part" + } +} + +class DependencyDownloaderHTTPResponseException(val url: URL, val responseCode: Int) : + IOException("Server returned HTTP response code: $responseCode for URL: $url") diff --git a/konan/utils/src/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt b/konan/utils/src/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt new file mode 100644 index 00000000000..f0ade14d1fb --- /dev/null +++ b/konan/utils/src/org/jetbrains/kotlin/konan/util/DependencyExtractor.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.util + +import org.jetbrains.kotlin.konan.file.unzipTo +import java.io.File + +internal class DependencyExtractor { + private val useZip = System.getProperty("os.name").startsWith("Windows") + + internal val archiveExtension = if (useZip) { + "zip" + } else { + "tar.gz" + } + + private fun extractTarGz(tarGz: File, targetDirectory: File) { + val tarProcess = ProcessBuilder().apply { + command("tar", "-xzf", tarGz.canonicalPath) + directory(targetDirectory) + }.start() + tarProcess.waitFor() + if (tarProcess.exitValue() != 0) { + throw RuntimeException("Cannot extract archive with dependency: ${tarGz.canonicalPath}") + } + } + + fun extract(archive: File, targetDirectory: File) { + if (useZip) { + archive.toPath().unzipTo(targetDirectory.toPath()) + } else { + extractTarGz(archive, targetDirectory) + } + } +} \ No newline at end of file diff --git a/konan/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt b/konan/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt new file mode 100644 index 00000000000..68d6e79bb54 --- /dev/null +++ b/konan/utils/src/org/jetbrains/kotlin/konan/util/DependencyProcessor.kt @@ -0,0 +1,284 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.util + +import org.jetbrains.kotlin.konan.file.use +import org.jetbrains.kotlin.konan.properties.Properties +import org.jetbrains.kotlin.konan.properties.propertyList +import java.io.File +import java.io.FileNotFoundException +import java.io.RandomAccessFile +import java.net.InetAddress +import java.net.URL +import java.net.UnknownHostException +import java.nio.file.Paths +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +private val Properties.dependenciesUrl: String + get() = getProperty("dependenciesUrl") + ?: throw IllegalStateException("No such property in konan.properties: dependenciesUrl") + +private val Properties.airplaneMode: Boolean + get() = getProperty("airplaneMode")?.toBoolean() ?: false + +private val Properties.downloadingAttempts: Int + get() = getProperty("downloadingAttempts")?.toInt() + ?: DependencyDownloader.DEFAULT_MAX_ATTEMPTS + +private val Properties.downloadingAttemptIntervalMs: Long + get() = getProperty("downloadingAttemptPauseMs")?.toLong() + ?: DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS + +private fun Properties.findCandidates(dependencies: List): Map> { + val dependencyProfiles = this.propertyList("dependencyProfiles") + return dependencies.map { dependency -> + dependency to dependencyProfiles.flatMap { profile -> + val candidateSpecs = propertyList("$dependency.$profile") + if (profile == "default" && candidateSpecs.isEmpty()) { + listOf(DependencySource.Remote.Public) + } else { + candidateSpecs.map { candidateSpec -> + when (candidateSpec) { + "remote:public" -> DependencySource.Remote.Public + "remote:internal" -> DependencySource.Remote.Internal + else -> DependencySource.Local(File(candidateSpec)) + } + } + } + } + }.toMap() +} + +sealed class DependencySource { + data class Local(val path: File) : DependencySource() + + sealed class Remote : DependencySource() { + object Public : Remote() + object Internal : Remote() + } +} + +/** + * Inspects dependencies and downloads all the missing ones into [dependenciesDirectory] from [dependenciesUrl]. + * If [airplaneMode] is true will throw a RuntimeException instead of downloading. + */ +class DependencyProcessor( + dependenciesRoot: File, + val dependenciesUrl: String, + dependencyToCandidates: Map>, + homeDependencyCache: File = defaultDependencyCacheDir, + val airplaneMode: Boolean = false, + maxAttempts: Int = DependencyDownloader.DEFAULT_MAX_ATTEMPTS, + attemptIntervalMs: Long = DependencyDownloader.DEFAULT_ATTEMPT_INTERVAL_MS, + customProgressCallback: ProgressCallback? = null, + val keepUnstable: Boolean = true +) { + + val dependenciesDirectory = dependenciesRoot.apply { mkdirs() } + val cacheDirectory = homeDependencyCache.apply { mkdirs() } + + val lockFile = File(cacheDirectory, ".lock").apply { if (!exists()) createNewFile() } + + var showInfo = true + private var isInfoShown = false + + private val downloader = DependencyDownloader(maxAttempts, attemptIntervalMs, customProgressCallback) + private val extractor = DependencyExtractor() + + private val archiveExtension get() = extractor.archiveExtension + + constructor( + dependenciesRoot: File, + properties: Properties, + dependencies: List, + dependenciesUrl: String = properties.dependenciesUrl, + keepUnstable: Boolean = true + ) : this( + dependenciesRoot, + dependenciesUrl, + dependencyToCandidates = properties.findCandidates(dependencies), + airplaneMode = properties.airplaneMode, + maxAttempts = properties.downloadingAttempts, + attemptIntervalMs = properties.downloadingAttemptIntervalMs, + keepUnstable = keepUnstable + ) + + class DependencyFile(directory: File, fileName: String) { + val file = File(directory, fileName).apply { createNewFile() } + private val dependencies = file.readLines().toMutableSet() + + fun contains(dependency: String) = dependencies.contains(dependency) + fun add(dependency: String) = dependencies.add(dependency) + fun remove(dependency: String) = dependencies.remove(dependency) + + fun removeAndSave(dependency: String) { + remove(dependency) + save() + } + + fun addAndSave(dependency: String) { + add(dependency) + save() + } + + fun save() { + val writer = file.writer() + writer.use { + dependencies.forEach { + writer.write(it) + writer.write("\n") + } + } + } + } + + private fun downloadDependency(dependency: String, baseUrl: String) { + val depDir = File(dependenciesDirectory, dependency) + val depName = depDir.name + + val fileName = "$depName.$archiveExtension" + val archive = cacheDirectory.resolve(fileName) + val url = URL("$baseUrl/$fileName") + + val extractedDependencies = DependencyFile(dependenciesDirectory, ".extracted") + if (extractedDependencies.contains(depName) && + depDir.exists() && + depDir.isDirectory && + depDir.list().isNotEmpty() + ) { + + if (!keepUnstable && depDir.list().contains(".unstable")) { + // The downloaded version of the dependency is unstable -> redownload it. + depDir.deleteRecursively() + archive.delete() + extractedDependencies.removeAndSave(dependency) + } else { + return + } + } + + if (showInfo && !isInfoShown) { + println("Downloading native dependencies (LLVM, sysroot etc). This is a one-time action performed only on the first run of the compiler.") + isInfoShown = true + } + + if (!archive.exists()) { + if (airplaneMode) { + throw FileNotFoundException( + """ + Cannot find a dependency locally: $dependency. + Set `airplaneMode = false` in konan.properties to download it. + """.trimIndent() + ) + } + downloader.download(url, archive) + } + println("Extracting dependency: $archive into $dependenciesDirectory") + extractor.extract(archive, dependenciesDirectory) + extractedDependencies.addAndSave(depName) + } + + companion object { + private val lock = ReentrantLock() + + val localKonanDir: File by lazy { + File(System.getenv("KONAN_DATA_DIR") ?: (System.getProperty("user.home") + File.separator + ".konan")) + } + + @JvmStatic + val defaultDependenciesRoot: File + get() = localKonanDir.resolve("dependencies") + + val defaultDependencyCacheDir: File + get() = localKonanDir.resolve("cache") + } + + private val resolvedDependencies = dependencyToCandidates.map { (dependency, candidates) -> + val candidate = candidates.asSequence().mapNotNull { candidate -> + when (candidate) { + is DependencySource.Local -> candidate.takeIf { it.path.exists() } + DependencySource.Remote.Public -> candidate + DependencySource.Remote.Internal -> candidate.takeIf { InternalServer.isAvailable } + } + }.firstOrNull() + + candidate ?: error("$dependency is not available; candidates:\n${candidates.joinToString("\n")}") + + dependency to candidate + }.toMap() + + private fun resolveDependency(dependency: String): File { + val candidate = resolvedDependencies[dependency] + return when (candidate) { + is DependencySource.Local -> candidate.path + is DependencySource.Remote -> File(dependenciesDirectory, dependency) + null -> error("$dependency not declared as dependency") + } + } + + fun resolveRelative(relative: String): File { + val path = Paths.get(relative) + if (path.isAbsolute) error("not a relative path: $relative") + + val dependency = path.first().toString() + return resolveDependency(dependency).let { + if (path.nameCount > 1) { + it.toPath().resolve(path.subpath(1, path.nameCount)).toFile() + } else { + it + } + } + } + + fun run() = lock.withLock { + RandomAccessFile(lockFile, "rw").channel.lock().use { + resolvedDependencies.forEach { (dependency, candidate) -> + val baseUrl = when (candidate) { + is DependencySource.Local -> null + DependencySource.Remote.Public -> dependenciesUrl + DependencySource.Remote.Internal -> InternalServer.url + } + // TODO: consider using different caches for different remotes. + if (baseUrl != null) { + downloadDependency(dependency, baseUrl) + } + } + } + } +} + +internal object InternalServer { + private val host = "repo.labs.intellij.net" + val url = "http://$host/kotlin-native" + + private val internalDomain = "labs.intellij.net" + + val isAvailable by lazy { + val envKey = "KONAN_USE_INTERNAL_SERVER" + val envValue = System.getenv(envKey) + when (envValue) { + "0" -> false + "1" -> true + null -> checkAccessible() + else -> error("unexpected environment: $envKey=$envValue") + } + } + + private fun checkAccessible(): Boolean { + if (!InetAddress.getLocalHost().canonicalHostName.endsWith(".$internalDomain")) { + // Fast path: + return false + } + + return try { + InetAddress.getByName(host) + true + } catch (e: UnknownHostException) { + false + } + } +} diff --git a/prepare/idea-plugin/build.gradle.kts b/prepare/idea-plugin/build.gradle.kts index c061426594c..5977ea9f79f 100644 --- a/prepare/idea-plugin/build.gradle.kts +++ b/prepare/idea-plugin/build.gradle.kts @@ -34,8 +34,10 @@ val projectsToShadow by extra(listOf( ":compiler:frontend.script", ":idea:ide-common", ":idea", + ":idea:idea-native", ":idea:idea-core", ":idea:idea-gradle", + //":idea:idea-gradle:native", //":idea-ultimate", ":compiler:ir.psi2ir", ":compiler:ir.tree", @@ -43,6 +45,9 @@ val projectsToShadow by extra(listOf( ":js:js.frontend", ":js:js.parser", ":js:js.serializer", + ":konan:konan-serializer", + ":konan:konan-metadata", + ":konan:konan-utils", ":compiler:light-classes", ":compiler:plugin-api", ":kotlin-preloader", diff --git a/settings.gradle b/settings.gradle index f96b3d54bec..24c0f00e326 100644 --- a/settings.gradle +++ b/settings.gradle @@ -77,6 +77,8 @@ include ":kotlin-build-common", ":idea:idea-android", ":idea:idea-android-output-parser", ":idea:idea-test-framework", + ":idea:idea-native", + //":idea:idea-gradle:native", ":idea", ":idea-runner", ":eval4j",