K2 Scripting: implement basic metadata serialization support

#KT-62305 fixed
NB: kotlin reflection do not see script class constructor after
this change, and it's ok, since the fact that the script is compiled
into a class is an implementation detail.
If needed, java reflection could be used to access the constructor.

compiler/fir/fir-serialization/src/org/jetbrains/kotlin/fir/serialization/FirElementSerializer.kt

@@ -37,6 +37,7 @@ import org.jetbrains.kotlin.fir.resolve.fullyExpandedType
 import org.jetbrains.kotlin.fir.resolve.providers.symbolProvider
 import org.jetbrains.kotlin.fir.resolve.toSymbol
 import org.jetbrains.kotlin.fir.scopes.*
+import org.jetbrains.kotlin.fir.scopes.impl.FirScriptDeclarationsScope
 import org.jetbrains.kotlin.fir.scopes.impl.nestedClassifierScope
 import org.jetbrains.kotlin.fir.serialization.constant.hasConstantValue
 import org.jetbrains.kotlin.fir.serialization.constant.toConstantValue
@@ -293,6 +294,88 @@ class FirElementSerializer private constructor(
         return builder
     }
 
+    fun scriptProto(script: FirScript): ProtoBuf.Class.Builder = whileAnalysing(session, script) {
+        val builder = ProtoBuf.Class.newBuilder()
+
+        val flags = Flags.getClassFlags(
+            extension.hasAdditionalAnnotations(script),
+            ProtoEnumFlags.visibility(Visibilities.Public),
+            ProtoEnumFlags.modality(Modality.FINAL),
+            ProtoEnumFlags.classKind(ClassKind.CLASS, false),
+            /* inner = */ false,
+            /* isData = */ false,
+            /* isExternal = */ false,
+            /* isExpect = */ false,
+            /* isValue = */ false,
+            /* isFun = */ false,
+            /* hasEnumEntries = */ false,
+        )
+        if (flags != builder.flags) {
+            builder.flags = flags
+        }
+
+        val name = script.name.let {
+            if (it.isSpecial) {
+                NameUtils.getScriptNameForFile(it.asStringStripSpecialMarkers().removePrefix("script-"))
+            } else it
+        }
+        val classId = ClassId(script.symbol.fqName.parentOrNull() ?: FqName.ROOT, name)
+
+        builder.fqName = getClassifierId(classId)
+
+        val memberScope = FirScriptDeclarationsScope(session, script)
+
+        val callableMembers = buildList {
+            memberScope.processAllCallables { add(it.fir) }
+        }
+
+        for (declaration in callableMembers) {
+            when (declaration) {
+                is FirProperty -> propertyProto(declaration)?.let { builder.addProperty(it) }
+                is FirSimpleFunction -> functionProto(declaration)?.let { builder.addFunction(it) }
+                else -> {}
+            }
+        }
+
+        memberScope.getClassifierNames().forEach { classifierName ->
+            memberScope.processClassifiersByName(classifierName) { nestedClassifier ->
+                when (nestedClassifier) {
+                    is FirRegularClassSymbol -> {
+                        builder.addNestedClassName(getSimpleNameIndex(nestedClassifier.name))
+                    }
+                    else -> {}
+                }
+            }
+        }
+
+        for (contextReceiver in script.contextReceivers) {
+            val typeRef = contextReceiver.typeRef
+            if (useTypeTable()) {
+                builder.addContextReceiverTypeId(typeId(typeRef))
+            } else {
+                builder.addContextReceiverType(typeProto(contextReceiver.typeRef))
+            }
+        }
+
+        if (versionRequirementTable == null) error("Version requirements must be serialized for scripts: ${script.render()}")
+
+        builder.addAllVersionRequirement(versionRequirementTable.serializeVersionRequirements(script))
+
+        extension.serializeScript(script, builder, versionRequirementTable, this)
+
+        if (metDefinitelyNotNullType) {
+            builder.addVersionRequirement(
+                writeLanguageVersionRequirement(LanguageFeature.DefinitelyNonNullableTypes, versionRequirementTable)
+            )
+        }
+
+        typeTable.serialize()?.let { builder.typeTable = it }
+        versionRequirementTable.serialize()?.let { builder.versionRequirementTable = it }
+
+        return builder
+    }
+
+
     /*
      * Order of nested classifiers:
      *   - declared classifiers in declaration order
@@ -1237,6 +1320,25 @@ class FirElementSerializer private constructor(
             return serializer
         }
 
+        @JvmStatic
+        fun createForScript(
+            session: FirSession,
+            scopeSession: ScopeSession,
+            script: FirScript,
+            extension: FirSerializerExtension,
+            typeApproximator: AbstractTypeApproximator,
+            languageVersionSettings: LanguageVersionSettings,
+            produceHeaderKlib: Boolean = false,
+        ): FirElementSerializer =
+            FirElementSerializer(
+                session, scopeSession, script,
+                Interner(), extension, MutableTypeTable(), MutableVersionRequirementTable(),
+                serializeTypeTableToFunction = false,
+                typeApproximator,
+                languageVersionSettings,
+                produceHeaderKlib,
+            )
+
         private fun writeLanguageVersionRequirement(
             languageFeature: LanguageFeature,
             versionRequirementTable: MutableVersionRequirementTable

compiler/fir/fir-serialization/src/org/jetbrains/kotlin/fir/serialization/FirSerializerExtension.kt

@@ -54,6 +54,14 @@ abstract class FirSerializerExtension {
     ) {
     }
 
+    open fun serializeScript(
+        script: FirScript,
+        proto: ProtoBuf.Class.Builder,
+        versionRequirementTable: MutableVersionRequirementTable,
+        childSerializer: FirElementSerializer
+    ) {
+    }
+
     open fun serializeConstructor(
         constructor: FirConstructor,
         proto: ProtoBuf.Constructor.Builder,

compiler/fir/fir-serialization/src/org/jetbrains/kotlin/fir/serialization/FirSerializerExtensionBase.kt

@@ -13,6 +13,7 @@ import org.jetbrains.kotlin.fir.declarations.*
 import org.jetbrains.kotlin.fir.expressions.FirAnnotation
 import org.jetbrains.kotlin.fir.serialization.constant.toConstantValue
 import org.jetbrains.kotlin.metadata.ProtoBuf
+import org.jetbrains.kotlin.metadata.ProtoBuf.Class.Builder
 import org.jetbrains.kotlin.metadata.deserialization.Flags
 import org.jetbrains.kotlin.metadata.serialization.MutableVersionRequirementTable
 import org.jetbrains.kotlin.name.FqName
@@ -37,6 +38,13 @@ abstract class FirSerializerExtensionBase(
         klass.serializeAnnotations(proto, protocol.classAnnotation)
     }
 
+    override fun serializeScript(
+        script: FirScript,
+        proto: Builder,
+        versionRequirementTable: MutableVersionRequirementTable,
+        childSerializer: FirElementSerializer,
+    ) {}
+
     override fun serializeConstructor(
         constructor: FirConstructor,
         proto: ProtoBuf.Constructor.Builder,

compiler/fir/fir2ir/jvm-backend/src/org/jetbrains/kotlin/fir/backend/jvm/FirJvmSerializerExtension.kt

@@ -113,6 +113,19 @@ class FirJvmSerializerExtension(
         }
     }
 
+    override fun serializeScript(
+        script: FirScript,
+        proto: ProtoBuf.Class.Builder,
+        versionRequirementTable: MutableVersionRequirementTable,
+        childSerializer: FirElementSerializer
+    ) {
+        assert((metadata as FirMetadataSource.Script).fir == script)
+        if (moduleName != JvmProtoBufUtil.DEFAULT_MODULE_NAME) {
+            proto.setExtension(JvmProtoBuf.classModuleName, stringTable.getStringIndex(moduleName))
+        }
+        writeLocalProperties(proto, JvmProtoBuf.classLocalVariable)
+    }
+
     // Interfaces which have @JvmDefault members somewhere in the hierarchy need the compiler 1.2.40+
     // so that the generated bridges in subclasses would call the super members correctly
     private fun writeVersionRequirementForJvmDefaultIfNeeded(

compiler/fir/fir2ir/jvm-backend/src/org/jetbrains/kotlin/fir/backend/jvm/FirMetadataSerializer.kt

@@ -135,6 +135,7 @@ class FirMetadataSerializer(
                 }
                 serializer!!.functionProto(withTypeParameters)?.build()
             }
+            is FirMetadataSource.Script -> serializer!!.scriptProto(metadata.fir).build()
             else -> null
         } ?: return null
         return message to serializer!!.stringTable as JvmStringTable
@@ -205,6 +206,13 @@ internal fun makeElementSerializer(
             approximator,
             languageVersionSettings,
         )
+        is FirMetadataSource.Script -> FirElementSerializer.createForScript(
+            session, scopeSession,
+            metadata.fir,
+            serializerExtension,
+            approximator,
+            languageVersionSettings
+        )
         else -> null
     }
 

compiler/fir/fir2ir/src/org/jetbrains/kotlin/fir/backend/Fir2IrAnnotationsFromPluginRegistrar.kt

@@ -103,6 +103,7 @@ class Fir2IrAnnotationsFromPluginRegistrar(private val components: Fir2IrCompone
                     ?.topmostParentClassId
                     ?.toSymbol(session)
                     ?.fir
+                is FirScript -> this
                 else -> error("Unsupported declaration type: $this")
             } ?: this
         }

compiler/ir/backend.jvm/lower/src/org/jetbrains/kotlin/backend/jvm/lower/ScriptLowering.kt

@@ -93,7 +93,7 @@ private class ScriptsToClassesLowering(val context: JvmBackendContext, val inner
             origin = IrDeclarationOrigin.SCRIPT_CLASS
             name = irScript.name.let {
                 if (it.isSpecial) {
-                    NameUtils.getScriptNameForFile(irScript.name.asStringStripSpecialMarkers().removePrefix("script-"))
+                    NameUtils.getScriptNameForFile(it.asStringStripSpecialMarkers().removePrefix("script-"))
                 } else it
             }
             kind = ClassKind.CLASS

compiler/testData/codegen/script/localDelegatedPropertyInLambda.kts

@@ -1,4 +1,3 @@
-// IGNORE_BACKEND_K2: JVM_IR
 // WITH_REFLECT
 
 import kotlin.reflect.KProperty

libraries/scripting/jvm-host-test/test/kotlin/script/experimental/jvmhost/test/ImplicitsFromScriptResultTest.kt

@@ -35,7 +35,7 @@ import kotlin.script.experimental.jvmhost.JvmScriptCompiler
  */
 class ImplicitsFromScriptResultTest : TestCase() {
 
-    fun testImplicits() = expectTestToFailOnK2 {
+    fun testImplicits() {
         // the implementation of the Compiler Host doesn't work with IR - the inter-script symbol table
         // should be maintained to make it run (see latest REPL compiler implementations for details
         // TODO: consider either fix it or rewrite to the REPL compiler

libraries/tools/kotlinp/test/org/jetbrains/kotlin/kotlinp/test/GenerateKotlinpTests.kt

@@ -16,7 +16,7 @@ fun main(args: Array<String>) {
                 model("")
             }
             testClass<AbstractK2KotlinpTest> {
-                model("")
+                model("", pattern = "^(.*)\\.kts?$")
             }
         }
     }

libraries/tools/kotlinp/test/org/jetbrains/kotlin/kotlinp/test/K2KotlinpTestGenerated.java

@@ -26,7 +26,7 @@ public class K2KotlinpTestGenerated extends AbstractK2KotlinpTest {
     }
 
     public void testAllFilesPresentInTestData() throws Exception {
-        KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("libraries/tools/kotlinp/testData"), Pattern.compile("^(.+)\\.kt$"), null, true);
+        KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("libraries/tools/kotlinp/testData"), Pattern.compile("^(.*)\\.kts?$"), null, true);
     }
 
     @TestMetadata("Annotations.kt")
@@ -109,6 +109,11 @@ public class K2KotlinpTestGenerated extends AbstractK2KotlinpTest {
         runTest("libraries/tools/kotlinp/testData/Properties.kt");
     }
 
+    @TestMetadata("scriptSimple.kts")
+    public void testScriptSimple() throws Exception {
+        runTest("libraries/tools/kotlinp/testData/scriptSimple.kts");
+    }
+
     @TestMetadata("SimpleClass.kt")
     public void testSimpleClass() throws Exception {
         runTest("libraries/tools/kotlinp/testData/SimpleClass.kt");
@@ -163,7 +168,7 @@ public class K2KotlinpTestGenerated extends AbstractK2KotlinpTest {
         }
 
         public void testAllFilesPresentInJvmDefault() throws Exception {
-            KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("libraries/tools/kotlinp/testData/jvmDefault"), Pattern.compile("^(.+)\\.kt$"), null, true);
+            KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("libraries/tools/kotlinp/testData/jvmDefault"), Pattern.compile("^(.*)\\.kts?$"), null, true);
         }
 
         @TestMetadata("withCompatibility.kt")
@@ -186,7 +191,7 @@ public class K2KotlinpTestGenerated extends AbstractK2KotlinpTest {
         }
 
         public void testAllFilesPresentInLocalClasses() throws Exception {
-            KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("libraries/tools/kotlinp/testData/localClasses"), Pattern.compile("^(.+)\\.kt$"), null, true);
+            KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("libraries/tools/kotlinp/testData/localClasses"), Pattern.compile("^(.*)\\.kts?$"), null, true);
         }
 
         @TestMetadata("AnonymousObject.kt")

libraries/tools/kotlinp/test/org/jetbrains/kotlin/kotlinp/test/KotlinpTestUtils.kt

@@ -95,7 +95,7 @@ private fun compileAndPrintAllFiles(
     }
 
     if (compareWithTxt) {
-        KotlinTestUtils.assertEqualsToFile(File(file.path.replace(".kt", ".txt")), main.toString())
+        KotlinTestUtils.assertEqualsToFile(File(file.path.replace(".kt.?".toRegex(), ".txt")), main.toString())
     }
 
     if (readWriteAndCompare && InTextDirectivesUtils.findStringWithPrefixes(file.readText(), "// NO_READ_WRITE_COMPARE") == null) {

libraries/tools/kotlinp/testData/scriptSimple.kts

@@ -0,0 +1,9 @@
+
+val a = 42
+
+class A(val p: String)
+
+fun A.foo() = p
+
+class B<T>(val v: T & Any)
+

libraries/tools/kotlinp/testData/scriptSimple.txt

@@ -0,0 +1,52 @@
+// ScriptSimple.class
+// ------------------------------------------
+public final class ScriptSimple {
+
+  // signature: foo(LScriptSimple$A;)Ljava/lang/String;
+  public final fun A.foo(): kotlin/String
+
+  // field: a:I
+  // getter: getA()I
+  public final val a: kotlin/Int /* = ... */
+    public final get
+
+  // nested class: A
+
+  // nested class: B
+
+  // module name: test-module
+}
+// ScriptSimple$A.class
+// ------------------------------------------
+public final class A : kotlin/Any {
+
+  // signature: <init>(Ljava/lang/String;)V
+  public constructor(p: kotlin/String)
+
+  // field: p:Ljava/lang/String;
+  // getter: getP()Ljava/lang/String;
+  public final val p: kotlin/String
+    public final get
+
+  // module name: test-module
+}
+// ScriptSimple$B.class
+// ------------------------------------------
+public final class B<T#0 /* T */> : kotlin/Any {
+
+  // requires language version 1.7.0 (level=ERROR)
+  // signature: <init>(Ljava/lang/Object;)V
+  public constructor(v: T#0 & Any)
+
+  // requires language version 1.7.0 (level=ERROR)
+  // field: v:Ljava/lang/Object;
+  // getter: getV()Ljava/lang/Object;
+  public final val v: T#0 & Any
+    public final get
+
+  // module name: test-module
+}
+// META-INF/test-module.kotlin_module
+// ------------------------------------------
+module {
+}

plugins/scripting/scripting-compiler/tests/org/jetbrains/kotlin/scripting/compiler/plugin/ScriptingCompilerPluginTest.kt

@@ -103,8 +103,7 @@ class ScriptingCompilerPluginTest : TestCase() {
         )
     }
 
-    // muted, see KT-61490
+    fun testLazyScriptDefinitionDiscovery() {
-    fun testLazyScriptDefinitionDiscovery() = expectTestToFailOnK2 {
 
         withTempDir { tmpdir ->
             withDisposable { disposable ->

plugins/scripting/scripting-compiler/tests/org/jetbrains/kotlin/scripting/compiler/test/ScriptCompilerTest.kt

@@ -10,10 +10,8 @@ import kotlinx.coroutines.runBlocking
 import org.jetbrains.kotlin.scripting.compiler.plugin.expectTestToFailOnK2
 import org.jetbrains.kotlin.scripting.compiler.plugin.getBaseCompilerArgumentsFromProperty
 import org.jetbrains.kotlin.scripting.compiler.plugin.impl.ScriptJvmCompilerIsolated
-import org.jetbrains.kotlin.utils.tryConstructClassFromStringArgs
 import kotlin.reflect.KClass
 import kotlin.reflect.KProperty
-import kotlin.reflect.full.createInstance
 import kotlin.reflect.full.declaredMembers
 import kotlin.script.experimental.api.*
 import kotlin.script.experimental.host.toScriptSource
@@ -43,8 +41,8 @@ class ScriptCompilerTest : TestCase() {
             """.trimIndent().toScriptSource()
         )
 
-        val kclass = res.valueOrThrow()
+        val kclass = res.valueOrThrow().java
-        val scriptInstance = kclass.createInstance()
+        val scriptInstance = kclass.constructors.first().newInstance()
         assertNotNull(scriptInstance)
     }
 
@@ -64,8 +62,8 @@ class ScriptCompilerTest : TestCase() {
             """.trimIndent().toScriptSource()
         )
 
-        val kclass = res.valueOrThrow()
+        val kclass = res.valueOrThrow().java
-        val scriptInstance = kclass.createInstance()
+        val scriptInstance = kclass.constructors.first().newInstance()
         assertNotNull(scriptInstance)
     }
 
@@ -97,8 +95,8 @@ class ScriptCompilerTest : TestCase() {
         )
 
         val kClass = res.valueOrThrow()
-        val scriptInstance = kClass.createInstance()
         val members = kClass.declaredMembers
+        val scriptInstance = kClass.java.constructors.first().newInstance()
         val namesToMembers = members.associateBy { it.name }
 
         fun prop(name: String) = namesToMembers[name]!! as KProperty<*>
@@ -112,6 +110,24 @@ class ScriptCompilerTest : TestCase() {
         assertNull(namesToMembers["_"])
     }
 
+    fun testSimpleReflection() {
+        val res = compileToClass(
+            """
+                val c = 3
+                class A {
+                    class B
+                }
+                val a = A()
+            """.trimIndent().toScriptSource()
+        )
+
+        val kClass = res.valueOrThrow()
+        val members = kClass.declaredMembers
+        assertEquals(2, members.size)
+        val nestedClasses = kClass.nestedClasses
+        assertEquals(1, nestedClasses.size)
+    }
+
     fun compile(
         script: SourceCode,
         cfgBody: ScriptCompilationConfiguration.Builder.() -> Unit

plugins/scripting/scripting-compiler/tests/org/jetbrains/kotlin/scripting/compiler/test/ScriptTemplateTest.kt

@@ -13,12 +13,11 @@ import org.jetbrains.kotlin.cli.common.messages.*
 import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.codegen.CompilationException
+import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.JVMConfigurationKeys
 import org.jetbrains.kotlin.script.loadScriptingPlugin
-import org.jetbrains.kotlin.scripting.compiler.plugin.TestMessageCollector
+import org.jetbrains.kotlin.scripting.compiler.plugin.*
-import org.jetbrains.kotlin.scripting.compiler.plugin.assertHasMessage
+import org.jetbrains.kotlin.scripting.compiler.plugin.impl.SCRIPT_BASE_COMPILER_ARGUMENTS_PROPERTY
-import org.jetbrains.kotlin.scripting.compiler.plugin.expectTestToFailOnK2
-import org.jetbrains.kotlin.scripting.compiler.plugin.updateWithBaseCompilerArguments
 import org.jetbrains.kotlin.scripting.configuration.ScriptingConfigurationKeys
 import org.jetbrains.kotlin.scripting.definitions.KotlinScriptDefinition
 import org.jetbrains.kotlin.scripting.definitions.ScriptDefinition
@@ -120,8 +119,8 @@ class ScriptTemplateTest : TestCase() {
         assertEqualsTrimmed(NUM_4_LINE + FIB_SCRIPT_OUTPUT_TAIL, out)
     }
 
-    // Should fail on K2, see KT-60452
+    // Fails on K2, see KT-60452
-    fun testScriptWithoutParams() {
+    fun testScriptWithoutParams() = expectTestToFailOnK2 {
         val messageCollector = TestMessageCollector()
         val aClass = compileScript("without_params.kts", ScriptWithoutParams::class, null, messageCollector = messageCollector)
         Assert.assertNotNull("Compilation failed:\n$messageCollector", aClass)
@@ -235,7 +234,8 @@ class ScriptTemplateTest : TestCase() {
         }
     }
 
-    fun testScriptWithParamConversion() {
+    // Fails on K2, see KT-62413
+    fun testScriptWithParamConversion() = expectTestToFailOnK2 {
         val messageCollector = TestMessageCollector()
         val aClass = compileScript("fib.kts", ScriptWithIntParam::class, messageCollector = messageCollector)
         Assert.assertNotNull("Compilation failed:\n$messageCollector", aClass)
@@ -383,6 +383,13 @@ class ScriptTemplateTest : TestCase() {
             configuration.put(JVMConfigurationKeys.DISABLE_STANDARD_SCRIPT_DEFINITION, true)
             configuration.put(JVMConfigurationKeys.RETAIN_OUTPUT_IN_MEMORY, true)
 
+            val isK2 = System.getProperty(SCRIPT_BASE_COMPILER_ARGUMENTS_PROPERTY)?.contains("-language-version 1.9") != true &&
+                    System.getProperty(SCRIPT_TEST_BASE_COMPILER_ARGUMENTS_PROPERTY)?.contains("-language-version 1.9") != true
+
+            if (isK2) {
+                configuration.put(CommonConfigurationKeys.USE_FIR, true)
+            }
+
             loadScriptingPlugin(configuration)
 
             val environment = KotlinCoreEnvironment.createForTests(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)