Implement script dependencies resolution directly from classloader

#KT-27956 fixed

core/descriptors.runtime/src/org/jetbrains/kotlin/descriptors/runtime/components/RuntimeModuleData.kt

@@ -31,10 +31,7 @@ import org.jetbrains.kotlin.load.java.components.JavaPropertyInitializerEvaluato
 import org.jetbrains.kotlin.load.java.components.JavaResolverCache
 import org.jetbrains.kotlin.load.java.components.SamConversionResolver
 import org.jetbrains.kotlin.load.java.components.SignaturePropagator
-import org.jetbrains.kotlin.load.java.lazy.JavaResolverComponents
+import org.jetbrains.kotlin.load.java.lazy.*
-import org.jetbrains.kotlin.load.java.lazy.JavaResolverSettings
-import org.jetbrains.kotlin.load.java.lazy.LazyJavaPackageFragmentProvider
-import org.jetbrains.kotlin.load.java.lazy.SingleModuleClassResolver
 import org.jetbrains.kotlin.load.java.typeEnhancement.SignatureEnhancement
 import org.jetbrains.kotlin.load.kotlin.*
 import org.jetbrains.kotlin.name.Name
@@ -43,6 +40,7 @@ import org.jetbrains.kotlin.serialization.deserialization.ContractDeserializer
 import org.jetbrains.kotlin.serialization.deserialization.DeserializationComponents
 import org.jetbrains.kotlin.serialization.deserialization.DeserializationConfiguration
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
+import org.jetbrains.kotlin.storage.StorageManager
 import org.jetbrains.kotlin.types.checker.NewKotlinTypeChecker
 import org.jetbrains.kotlin.utils.Jsr305State
 
@@ -59,35 +57,29 @@ class RuntimeModuleData private constructor(
             val module = ModuleDescriptorImpl(Name.special("<runtime module for $classLoader>"), storageManager, builtIns)
             builtIns.builtInsModule = module
 
+            builtIns.initialize(module, isAdditionalBuiltInsFeatureSupported = true)
+
             val reflectKotlinClassFinder = ReflectKotlinClassFinder(classLoader)
             val deserializedDescriptorResolver = DeserializedDescriptorResolver()
             val singleModuleClassResolver = SingleModuleClassResolver()
             val notFoundClasses = NotFoundClasses(storageManager, module)
-            val annotationTypeQualifierResolver = AnnotationTypeQualifierResolver(storageManager, Jsr305State.DISABLED)
-            val javaResolverComponents = JavaResolverComponents(
-                storageManager, ReflectJavaClassFinder(classLoader), reflectKotlinClassFinder, deserializedDescriptorResolver,
-                SignaturePropagator.DO_NOTHING, RuntimeErrorReporter, JavaResolverCache.EMPTY,
-                JavaPropertyInitializerEvaluator.DoNothing, SamConversionResolver.Empty, RuntimeSourceElementFactory,
-                singleModuleClassResolver, PackagePartProvider.Empty, SupertypeLoopChecker.EMPTY, LookupTracker.DO_NOTHING, module,
-                ReflectionTypes(module, notFoundClasses), annotationTypeQualifierResolver,
-                SignatureEnhancement(annotationTypeQualifierResolver, Jsr305State.DISABLED),
-                JavaClassesTracker.Default, JavaResolverSettings.Default, NewKotlinTypeChecker.Default
-            )
 
-            val lazyJavaPackageFragmentProvider = LazyJavaPackageFragmentProvider(javaResolverComponents)
+            val lazyJavaPackageFragmentProvider =
+                makeLazyJavaPackageFragmentFromClassLoaderProvider(
+                    classLoader, module, storageManager, notFoundClasses,
+                    reflectKotlinClassFinder, deserializedDescriptorResolver, singleModuleClassResolver
+                )
 
-            builtIns.initialize(module, isAdditionalBuiltInsFeatureSupported = true)
+            val deserializationComponentsForJava =
+                makeDeserializationComponentsForJava(
+                    module, storageManager, notFoundClasses, lazyJavaPackageFragmentProvider,
+                    reflectKotlinClassFinder, deserializedDescriptorResolver
+                )
+
+            deserializedDescriptorResolver.setComponents(deserializationComponentsForJava)
 
             val javaDescriptorResolver = JavaDescriptorResolver(lazyJavaPackageFragmentProvider, JavaResolverCache.EMPTY)
-            val javaClassDataFinder = JavaClassDataFinder(reflectKotlinClassFinder, deserializedDescriptorResolver)
+            singleModuleClassResolver.resolver = javaDescriptorResolver
-            val binaryClassAnnotationAndConstantLoader = BinaryClassAnnotationAndConstantLoaderImpl(
-                module, notFoundClasses, storageManager, reflectKotlinClassFinder
-            )
-            val deserializationComponentsForJava = DeserializationComponentsForJava(
-                storageManager, module, DeserializationConfiguration.Default, javaClassDataFinder,
-                binaryClassAnnotationAndConstantLoader, lazyJavaPackageFragmentProvider, notFoundClasses,
-                RuntimeErrorReporter, LookupTracker.DO_NOTHING, ContractDeserializer.DEFAULT, NewKotlinTypeChecker.Default
-            )
 
             // .kotlin_builtins files should be found by the same class loader that loaded stdlib classes
             val stdlibClassLoader = Unit::class.java.classLoader
@@ -96,9 +88,6 @@ class RuntimeModuleData private constructor(
                 DeserializationConfiguration.Default, NewKotlinTypeChecker.Default
             )
 
-            singleModuleClassResolver.resolver = javaDescriptorResolver
-            deserializedDescriptorResolver.setComponents(deserializationComponentsForJava)
-
             module.setDependencies(module)
             module.initialize(CompositePackageFragmentProvider(listOf(javaDescriptorResolver.packageFragmentProvider, builtinsProvider)))
 
@@ -109,3 +98,45 @@ class RuntimeModuleData private constructor(
         }
     }
 }
+
+fun makeLazyJavaPackageFragmentFromClassLoaderProvider(
+    classLoader: ClassLoader,
+    module: ModuleDescriptor,
+    storageManager: StorageManager,
+    notFoundClasses: NotFoundClasses,
+    reflectKotlinClassFinder: KotlinClassFinder,
+    deserializedDescriptorResolver: DeserializedDescriptorResolver,
+    singleModuleClassResolver: ModuleClassResolver
+): LazyJavaPackageFragmentProvider {
+    val annotationTypeQualifierResolver = AnnotationTypeQualifierResolver(storageManager, Jsr305State.DISABLED)
+    val javaResolverComponents = JavaResolverComponents(
+        storageManager, ReflectJavaClassFinder(classLoader), reflectKotlinClassFinder, deserializedDescriptorResolver,
+        SignaturePropagator.DO_NOTHING, RuntimeErrorReporter, JavaResolverCache.EMPTY,
+        JavaPropertyInitializerEvaluator.DoNothing, SamConversionResolver.Empty, RuntimeSourceElementFactory,
+        singleModuleClassResolver, PackagePartProvider.Empty, SupertypeLoopChecker.EMPTY, LookupTracker.DO_NOTHING, module,
+        ReflectionTypes(module, notFoundClasses), annotationTypeQualifierResolver,
+        SignatureEnhancement(annotationTypeQualifierResolver, Jsr305State.DISABLED),
+        JavaClassesTracker.Default, JavaResolverSettings.Default, NewKotlinTypeChecker.Default
+    )
+
+    return LazyJavaPackageFragmentProvider(javaResolverComponents)
+}
+
+fun makeDeserializationComponentsForJava(
+    module: ModuleDescriptor,
+    storageManager: StorageManager,
+    notFoundClasses: NotFoundClasses,
+    lazyJavaPackageFragmentProvider: LazyJavaPackageFragmentProvider,
+    reflectKotlinClassFinder: KotlinClassFinder,
+    deserializedDescriptorResolver: DeserializedDescriptorResolver
+): DeserializationComponentsForJava {
+    val javaClassDataFinder = JavaClassDataFinder(reflectKotlinClassFinder, deserializedDescriptorResolver)
+    val binaryClassAnnotationAndConstantLoader = BinaryClassAnnotationAndConstantLoaderImpl(
+        module, notFoundClasses, storageManager, reflectKotlinClassFinder
+    )
+    return DeserializationComponentsForJava(
+        storageManager, module, DeserializationConfiguration.Default, javaClassDataFinder,
+        binaryClassAnnotationAndConstantLoader, lazyJavaPackageFragmentProvider, notFoundClasses,
+        RuntimeErrorReporter, LookupTracker.DO_NOTHING, ContractDeserializer.DEFAULT, NewKotlinTypeChecker.Default
+    )
+}

libraries/scripting/jvm-host-test/build.gradle.kts

@@ -38,6 +38,7 @@ sourceSets {
 }
 
 projectTest(parallel = true) {
+    dependsOn(":dist")
     workingDir = rootDir
 }
 

libraries/scripting/jvm-host-test/test/kotlin/script/experimental/jvmhost/test/ResolveFromClassloaderTest.kt

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.jvmhost.test
+
+import junit.framework.TestCase
+import org.junit.Test
+import java.io.File
+import kotlin.script.experimental.api.ResultValue
+import kotlin.script.experimental.api.ScriptCompilationConfiguration
+import kotlin.script.experimental.api.dependencies
+import kotlin.script.experimental.api.valueOrThrow
+import kotlin.script.experimental.host.toScriptSource
+import kotlin.script.experimental.jvm.JvmDependency
+import kotlin.script.experimental.jvm.JvmDependencyFromClassLoader
+import kotlin.script.experimental.jvmhost.BasicJvmScriptingHost
+
+class ResolveFromClassloaderTest : TestCase() {
+
+    @Test
+    fun testResolveFromClassloader() {
+        val script = "${ShouldBeVisibleFromScript::class.java.name}().x".toScriptSource()
+        val compilationConfiguration = ScriptCompilationConfiguration {
+            dependencies(JvmDependencyFromClassLoader { ShouldBeVisibleFromScript::class.java.classLoader })
+        }
+        val res = BasicJvmScriptingHost().eval(script, compilationConfiguration, null).valueOrThrow().returnValue as ResultValue.Value
+        assertEquals(42, res.value)
+    }
+
+    @Test
+    fun testResolveFromClassloaderAndClassPath() {
+        val script = """
+            org.jetbrains.kotlin.mainKts.MainKtsConfigurator()
+            ${ShouldBeVisibleFromScript::class.java.name}().x
+        """.trimIndent().toScriptSource()
+        val classpath = listOf(
+            File("dist/kotlinc/lib/kotlin-main-kts.jar").also {
+                assertTrue("kotlin-main-kts.jar not found, run dist task: ${it.absolutePath}", it.exists())
+            }
+        )
+        val compilationConfiguration = ScriptCompilationConfiguration {
+            dependencies(
+                JvmDependencyFromClassLoader { ShouldBeVisibleFromScript::class.java.classLoader },
+                JvmDependency(classpath)
+            )
+        }
+        val res = BasicJvmScriptingHost().eval(script, compilationConfiguration, null).valueOrThrow().returnValue as ResultValue.Value
+        assertEquals(42, res.value)
+    }
+}
+
+@Suppress("unused")
+class ShouldBeVisibleFromScript {
+    val x = 42
+}

libraries/scripting/jvm/src/kotlin/script/experimental/jvm/impl/BridgeDependenciesResolver.kt

@@ -97,7 +97,7 @@ internal fun ScriptContents.toScriptSource(): SourceCode = when {
     else -> throw IllegalArgumentException("Unable to convert script contents $this into script source")
 }
 
-fun List<ScriptDependency>?.toClassPathOrEmpty() = this?.flatMap { (it as JvmDependency).classpath } ?: emptyList()
+fun List<ScriptDependency>?.toClassPathOrEmpty() = this?.flatMap { (it as? JvmDependency)?.classpath ?: emptyList() } ?: emptyList()
 
 internal fun List<SourceCode>?.toFilesOrEmpty() = this?.map {
     val externalSource = it as? ExternalSourceCode

libraries/scripting/jvm/src/kotlin/script/experimental/jvm/jvmScriptCompilation.kt

@@ -19,6 +19,12 @@ data class JvmDependency(val classpath: List<File>) : ScriptDependency {
     companion object { private const val serialVersionUID: Long = 1L }
 }
 
+typealias ClassLoaderByConfiguration = (ScriptCompilationConfiguration) -> ClassLoader
+
+class JvmDependencyFromClassLoader(val classLoaderGetter: ClassLoaderByConfiguration) : ScriptDependency {
+    fun getClassLoader(configuration: ScriptCompilationConfiguration): ClassLoader = classLoaderGetter(configuration)
+}
+
 interface JvmScriptCompilationConfigurationKeys
 
 open class JvmScriptCompilationConfigurationBuilder : PropertiesCollection.Builder(), JvmScriptCompilationConfigurationKeys {

plugins/scripting/scripting-compiler/build.gradle.kts

@@ -12,6 +12,7 @@ dependencies {
     compileOnly(project(":compiler:psi"))
     compileOnly(project(":compiler:plugin-api"))
     compileOnly(project(":compiler:cli"))
+    compileOnly(project(":core:descriptors.runtime"))
     compile(project(":kotlin-scripting-common"))
     compile(project(":kotlin-scripting-jvm"))
     compile(project(":kotlin-scripting-compiler-impl"))

plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/impl/PackageFragmentFromClassLoaderProvider.kt

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.compiler.plugin.impl
+
+import com.intellij.openapi.project.Project
+import org.jetbrains.kotlin.analyzer.ModuleInfo
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.descriptors.NotFoundClasses
+import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
+import org.jetbrains.kotlin.descriptors.runtime.components.*
+import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.load.java.lazy.SingleModuleClassResolver
+import org.jetbrains.kotlin.resolve.BindingTrace
+import org.jetbrains.kotlin.resolve.jvm.extensions.PackageFragmentProviderExtension
+import org.jetbrains.kotlin.storage.StorageManager
+import org.jetbrains.kotlin.load.kotlin.*
+import kotlin.script.experimental.api.ScriptCompilationConfiguration
+import kotlin.script.experimental.jvm.ClassLoaderByConfiguration
+
+
+class PackageFragmentFromClassLoaderProviderExtension(
+    val classLoaderGetter: ClassLoaderByConfiguration,
+    val scriptCompilationConfiguration: ScriptCompilationConfiguration
+) : PackageFragmentProviderExtension {
+
+    override fun getPackageFragmentProvider(
+        project: Project,
+        module: ModuleDescriptor,
+        storageManager: StorageManager,
+        trace: BindingTrace,
+        moduleInfo: ModuleInfo?,
+        lookupTracker: LookupTracker
+    ): PackageFragmentProvider? {
+        val classLoader = classLoaderGetter(scriptCompilationConfiguration)
+
+        val reflectKotlinClassFinder = ReflectKotlinClassFinder(classLoader)
+        val deserializedDescriptorResolver = DeserializedDescriptorResolver()
+        val singleModuleClassResolver = SingleModuleClassResolver()
+        val notFoundClasses = NotFoundClasses(storageManager, module)
+
+        val lazyJavaPackageFragmentProvider =
+            makeLazyJavaPackageFragmentFromClassLoaderProvider(
+                classLoader, module, storageManager, notFoundClasses,
+                reflectKotlinClassFinder, deserializedDescriptorResolver, singleModuleClassResolver
+            )
+
+        val deserializationComponentsForJava =
+            makeDeserializationComponentsForJava(
+                module, storageManager, notFoundClasses, lazyJavaPackageFragmentProvider,
+                reflectKotlinClassFinder, deserializedDescriptorResolver
+            )
+
+        deserializedDescriptorResolver.setComponents(deserializationComponentsForJava)
+
+        return lazyJavaPackageFragmentProvider
+    }
+}
+

plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/impl/ScriptJvmCompilerImpls.kt

@@ -18,12 +18,14 @@ import org.jetbrains.kotlin.codegen.state.GenerationState
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.resolve.jvm.extensions.PackageFragmentProviderExtension
 import org.jetbrains.kotlin.scripting.compiler.plugin.ScriptJvmCompilerProxy
 import org.jetbrains.kotlin.scripting.compiler.plugin.dependencies.ScriptsCompilationDependencies
 import org.jetbrains.kotlin.scripting.definitions.ScriptDependenciesProvider
 import kotlin.script.experimental.api.*
 import kotlin.script.experimental.host.ScriptingHostConfiguration
 import kotlin.script.experimental.jvm.JvmDependency
+import kotlin.script.experimental.jvm.JvmDependencyFromClassLoader
 import kotlin.script.experimental.jvm.compilationCache
 import kotlin.script.experimental.jvm.impl.KJvmCompiledScript
 import kotlin.script.experimental.jvm.jvm
@@ -138,6 +140,16 @@ private fun doCompile(
     messageCollector: ScriptDiagnosticsMessageCollector,
     getScriptConfiguration: (KtFile) -> ScriptCompilationConfiguration
 ): ResultWithDiagnostics<KJvmCompiledScript<Any>> {
+
+    context.baseScriptCompilationConfiguration[ScriptCompilationConfiguration.dependencies]?.forEach { dependency ->
+        if (dependency is JvmDependencyFromClassLoader) {
+            PackageFragmentProviderExtension.registerExtension(
+                context.environment.project,
+                PackageFragmentFromClassLoaderProviderExtension(dependency.classLoaderGetter, context.baseScriptCompilationConfiguration)
+            )
+        }
+    }
+
     val analysisResult = analyze(sourceFiles, context.environment)
 
     if (!analysisResult.shouldGenerateCode) return failure(

plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/impl/compilationContext.kt

@@ -185,7 +185,7 @@ private fun createInitialCompilerConfiguration(
         scriptCompilationConfiguration[ScriptCompilationConfiguration.dependencies]?.let { dependencies ->
             addJvmClasspathRoots(
                 dependencies.flatMap {
-                    (it as JvmDependency).classpath
+                    (it as? JvmDependency)?.classpath ?: emptyList()
                 }
             )
         }