Here goes stdlib separation.

$ gradlew backend.native:stdlib

builds you stdlib.kt.bc now.

The test runs are taught to supply
-library stdlib.kt.bc to kotlin compiler for proper imports resolution,
and then later stdlib.kt.bc is provided to clang for the final link step.

backend.native/build.gradle

@@ -161,19 +161,38 @@ dependencies {
 build.dependsOn 'compilerClasses','cli_bcClasses','bc_frontendClasses'
 
 
+task stdlib(type: JavaExec) {
+    main 'org.jetbrains.kotlin.cli.bc.K2NativeKt'
+    classpath configurations.cli_bc
+
+    args project(':runtime').file('src/main/kotlin')
+
+    args '-output', project(':runtime').file('build/stdlib.kt.bc')
+
+    jvmArgs '-ea', "-Djava.library.path=${project.buildDir}/nativelibs"
+
+    dependsOn ':runtime:build'
+
+    doFirst {
+    args '-runtime', project(':runtime').build.outputs.files.singleFile
+    }
+    environment 'LD_LIBRARY_PATH' : "${llvmDir}/lib:${project.buildDir}/nativelibs"
+}
+
 task run(type: JavaExec) {
    main 'org.jetbrains.kotlin.cli.bc.K2NativeKt'
    classpath configurations.cli_bc
 
    args 'tests/codegen/function/sum.kt'
 
-   args '-headers', project(':runtime').file('src/main/kotlin')
+   args '-library', project(':runtime').file('build/stdlib.kt.bc')
 
    args '-output', "$buildDir/out.bc"
 
    jvmArgs '-ea', "-Djava.library.path=${project.buildDir}/nativelibs"
 
    dependsOn ':runtime:build'
+   dependsOn 'stdlib'
 
    doFirst {
        args '-runtime', project(':runtime').build.outputs.files.singleFile

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt

@@ -37,25 +37,19 @@ class NativeAnalyzer(
 class K2Native : CLICompiler<K2NativeCompilerArguments>() { 
 
     val defaultModuleName = "main";
-    
+
     override fun doExecute(arguments     : K2NativeCompilerArguments,
                            configuration : CompilerConfiguration,
                            rootDisposable: Disposable
                           ): ExitCode {
 
-        configuration.put(CommonConfigurationKeys.MODULE_NAME, defaultModuleName )
+        configuration.put(CommonConfigurationKeys.MODULE_NAME, defaultModuleName)
 
         configuration.addKotlinSourceRoots(arguments.freeArgs)
 
-
+        val libraries = arguments.libraries?.asList<String>() ?: listOf<String>()
-        // TODO: remove, once linker is available.
+        configuration.put(KonanConfigurationKeys.LIBRARY_FILES, libraries)
-        if (arguments.headers != null) {
+        libraries.forEach{println(it)}
-            configuration.addKotlinSourceRoots(arguments.headers.asList())
-        }
-
-        // As of now we don't have any library mechnanism
-        var libraryFiles = listOf<String>()
-        configuration.put(KonanConfigurationKeys.LIBRARY_FILES, libraryFiles)
 
         val environment = KotlinCoreEnvironment.createForProduction(rootDisposable,
             configuration, Arrays.asList<String>("extensions/common.xml"))

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.java

@@ -14,9 +14,10 @@ public class K2NativeCompilerArguments extends CommonCompilerArguments {
     @ValueDescription("<path>")
     public String runtimeFile;
 
-    @Argument(value = "headers", description = "Header files used for compilation")
+    @Argument(value = "library", description = "Bitcode file with metadata attached")
     @ValueDescription("<path>")
-    public String[] headers;
+    public String[] libraries;
+
 
     public K2NativeCompilerArguments() {
     }
@@ -24,7 +25,7 @@ public class K2NativeCompilerArguments extends CommonCompilerArguments {
     public K2NativeCompilerArguments(K2NativeCompilerArguments arguments) {
         this.outputFile = arguments.outputFile;
         this.runtimeFile = arguments.runtimeFile;
-        this.headers = arguments.headers;
+        this.libraries = arguments.libraries;
     }
 
     public CommonCompilerArguments copy() {

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/KonanConfig.kt

@@ -23,14 +23,9 @@ import org.jetbrains.kotlin.backend.konan.llvm.KotlinKonanMetadataUtils
 class KonanConfig(val project: Project, val configuration: CompilerConfiguration) {
     private val storageManager = LockBasedStorageManager()
 
-    private val sourceFilesFromLibraries = listOf<KtFile>()
+    private val libraries = configuration.getList(KonanConfigurationKeys.LIBRARY_FILES)
-    private var metadata = listOf<KotlinKonanMetadata>()
-    internal var moduleDescriptors: MutableList<JsModuleDescriptor<ModuleDescriptorImpl>>? = null
 
-    init {
+    private val metadata = KotlinKonanMetadataUtils.loadLibMetadata(libraries)
-        val libraries = configuration.getList(KonanConfigurationKeys.LIBRARY_FILES)
-        metadata = KotlinKonanMetadataUtils.loadLibMetadata(libraries)
-    }
 
     val moduleId: String
         get() = configuration.getNotNull(CommonConfigurationKeys.MODULE_NAME)
@@ -38,26 +33,25 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
     val moduleKind: ModuleKind
         get() = configuration.get(KonanConfigurationKeys.MODULE_KIND)!!
 
-    fun getModuleDescriptors(): MutableList<JsModuleDescriptor<ModuleDescriptorImpl>> {
+    // We reuse JsModuleDescriptor for serialization for now, as we haven't got one for Konan yet.
-        if (moduleDescriptors != null) return moduleDescriptors!!
+    internal val moduleDescriptors: MutableList<JsModuleDescriptor<ModuleDescriptorImpl>> by lazy {
+
+        val jsDescriptors = mutableListOf<JsModuleDescriptor<ModuleDescriptorImpl>>()
+        val descriptors = mutableListOf<ModuleDescriptorImpl>()
 
-        moduleDescriptors = mutableListOf<JsModuleDescriptor<ModuleDescriptorImpl>>()
-        val kotlinModuleDescriptors = mutableListOf<ModuleDescriptorImpl>()
         for (metadataEntry in metadata) {
             val descriptor = createModuleDescriptor(metadataEntry)
-            moduleDescriptors!!.add(descriptor)
+            jsDescriptors.add(descriptor)
-            kotlinModuleDescriptors.add(descriptor.data)
+            descriptors.add(descriptor.data)
         }
 
-        for (module in moduleDescriptors!!) {
+        for (module in jsDescriptors) {
-            setDependencies(module.data, kotlinModuleDescriptors)
+            setDependencies(module.data, descriptors)
         }
 
-        return moduleDescriptors!!
+        jsDescriptors 
     }
 
-
-    // We reuse JsModuleDescriptor for serialization for now, as we haven't got one for Konan yet.
     private fun createModuleDescriptor(metadata: KotlinKonanMetadata): JsModuleDescriptor<ModuleDescriptorImpl> {
 
         val moduleDescriptor = ModuleDescriptorImpl(
@@ -77,14 +71,6 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
         private fun setDependencies(module: ModuleDescriptorImpl, modules: List<ModuleDescriptorImpl>) {
             module.setDependencies(modules.plus(KonanPlatform.builtIns.builtInsModule))
         }
-
-        fun withJsLibAdded(files: Collection<KtFile>, config: KonanConfig): Collection<KtFile> {
-            val allFiles = mutableListOf<KtFile>()
-            allFiles.addAll(files)
-            // We don't store source files in the bitcode for Konan, dont't we?
-            //allFiles.addAll(config.getSourceFilesFromLibraries());
-            return allFiles
-        }
     }
 
 }

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/TopDownAnalyzerFacadeForKonan.kt

@@ -38,7 +38,7 @@ object TopDownAnalyzerFacadeForKonan {
         // Make sure the compiler produced BuiltIns module comes in last
         context.setDependencies(
                 listOf(context.module) +
-                config.getModuleDescriptors().map { it.data } +
+                config.moduleDescriptors.map { it.data } +
                 listOf(compilerBuiltInsModule)
         )
         return analyzeFilesWithGivenTrace(files, BindingTraceContext(), context, config)
@@ -51,14 +51,12 @@ object TopDownAnalyzerFacadeForKonan {
             config: KonanConfig
     ): AnalysisResult {
 
-        val allFiles = KonanConfig.withJsLibAdded(files, config)
-
         // we print out each file we compile for now
-        allFiles.forEach{println(it)}
+        files.forEach{println(it)}
 
         val analyzerForKonan = createTopDownAnalyzerForKonan(
                 moduleContext, trace,
-                FileBasedDeclarationProviderFactory(moduleContext.storageManager, allFiles),
+                FileBasedDeclarationProviderFactory(moduleContext.storageManager, files),
                 config.configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS, LanguageVersionSettingsImpl.DEFAULT)
         )
 
@@ -66,9 +64,9 @@ object TopDownAnalyzerFacadeForKonan {
         return AnalysisResult.success(trace.bindingContext, moduleContext.module)
     }
 
-    fun checkForErrors(allFiles: Collection<KtFile>, bindingContext: BindingContext) {
+    fun checkForErrors(files: Collection<KtFile>, bindingContext: BindingContext) {
         AnalyzingUtils.throwExceptionOnErrors(bindingContext)
-        for (file in allFiles) {
+        for (file in files) {
             AnalyzingUtils.checkForSyntacticErrors(file)
         }
     }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -85,6 +85,13 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
         element.acceptChildrenVoid(this)
     }
 
+    //-------------------------------------------------------------------------//
+    override fun visitModuleFragment(module: IrModuleFragment) {
+        logger.log("visitModule                  : ${ir2string(module)}")
+        module.acceptChildrenVoid(this)
+        metadator.endModule(module)
+    }
+
     //-------------------------------------------------------------------------//
 
     override fun visitWhen(expression: IrWhen) {
@@ -183,6 +190,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
     //-------------------------------------------------------------------------//
 
     override fun visitFunction(declaration: IrFunction) {
+        logger.log("visitFunction                  : ${ir2string(declaration)}")
         codegen.function(declaration)
         metadator.function(declaration)
         declaration.acceptChildrenVoid(this)
@@ -191,6 +199,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
     //-------------------------------------------------------------------------//
 
     override fun visitClass(declaration: IrClass) {
+        logger.log("visitClass                  : ${ir2string(declaration)}")
         if (declaration.descriptor.kind == ClassKind.ANNOTATION_CLASS) {
             // do not generate any code for annotation classes as a workaround for NotImplementedError
             return

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/KotlinKonanMetadataUtils.kt

@@ -60,10 +60,8 @@ object KotlinKonanMetadataUtils {
     fun loadMetadata(file: File): List<KotlinKonanMetadata> {
         assert(file.exists()) { "Library $file not found" }
         val metadataList = arrayListOf<KotlinKonanMetadata>()
-
+        val content = readModuleMetadata(file)
-        // For the time being we don't have a reader, so just produce nothing
+        parseMetadata(content, metadataList)
-        //val content = readModuleMetadata(file)
-        //parseMetadata(content, metadataList)
 
         return metadataList
     }
@@ -115,3 +113,4 @@ object KotlinKonanMetadataUtils {
         return allMetadata;
     }
 }
+

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/MetadataGenerator.kt

@@ -4,9 +4,98 @@ package org.jetbrains.kotlin.backend.konan.llvm
 import kotlin_native.interop.*
 import llvm.*
 import org.jetbrains.kotlin.ir.declarations.IrFunction
+import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
 import org.jetbrains.kotlin.backend.konan.llvm.BinaryLinkdata.*
 import org.jetbrains.kotlin.types.KotlinType
 
+import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
+import org.jetbrains.kotlin.serialization.deserialization.DeserializationConfiguration
+import org.jetbrains.kotlin.serialization.js.*
+import org.jetbrains.kotlin.utils.*
+import java.io.*
+
+fun readModuleMetadata(file: File): String {
+
+    val reader = MetadataReader(file)
+    var metadataString: String? = null
+
+    try {
+        val metadataNode = reader.namedMetadataNode("kmetadata", 0);
+        val stringNode = reader.metadataOperand(metadataNode, 0)!!
+        metadataString = reader.string(stringNode)
+    } finally {
+        reader.close()
+    }
+
+    return metadataString!!
+}
+
+class MetadataReader(file: File) {
+
+    lateinit var llvmModule: LLVMOpaqueModule
+    lateinit var llvmContext: LLVMOpaqueContext
+
+    init {
+        memScoped {
+            val bufRef = alloc(LLVMOpaqueMemoryBuffer.ref)
+            val errorRef = alloc(Int8Box.ref)
+            val res = LLVMCreateMemoryBufferWithContentsOfFile(file.toString(), bufRef, errorRef)
+            if (res != 0) {
+                throw Error(errorRef.value?.asCString()?.toString())
+            }
+
+            llvmContext = LLVMContextCreate()!!
+            val moduleRef = alloc(LLVMOpaqueModule.ref)
+            val parseResult = LLVMParseBitcodeInContext2(llvmContext, bufRef.value, moduleRef)
+            if (parseResult != 0) {
+                throw Error(parseResult.toString())
+            }
+
+            llvmModule = moduleRef.value!!
+        }
+    }
+
+
+    fun string(node: LLVMOpaqueValue): String {
+        memScoped { 
+            val len = alloc(Int32Box)
+            val str1 = LLVMGetMDString(node, len)!!
+            val str = str1.asCString().toString() 
+            return str
+
+        }
+    }
+
+    fun namedMetadataNode(name: String, index: Int): LLVMOpaqueValue {
+        memScoped {
+            val nodeCount = LLVMGetNamedMetadataNumOperands(llvmModule, "kmetadata")!!
+            val nodeArray = alloc(array[nodeCount](Ref to LLVMOpaqueValue))
+
+            LLVMGetNamedMetadataOperands(llvmModule, "kmetadata", nodeArray[0])!!
+
+            return nodeArray[0].value!!
+        }
+    }
+
+    fun metadataOperand(metadataNode: LLVMOpaqueValue, index: Int): LLVMOpaqueValue {
+        memScoped {
+            val operandCount = LLVMGetMDNodeNumOperands(metadataNode)!!
+            val operandArray = alloc(array[operandCount](Ref to LLVMOpaqueValue))
+
+            LLVMGetMDNodeOperands(metadataNode, operandArray[0])!!
+
+            return operandArray[0].value!!
+        }
+    }
+
+    fun close() {
+        LLVMDisposeModule(llvmModule)
+        LLVMContextDispose(llvmContext)
+    }
+}
+
+
 internal class MetadataGenerator(override val context: Context): ContextUtils {
 
     private fun metadataString(str: String): LLVMOpaqueValue {
@@ -74,6 +163,44 @@ internal class MetadataGenerator(override val context: Context): ContextUtils {
         val md = metadataFun(fn, proto)
         emitModuleMetadata("kfun", md);
     }
+
+    // Quick check consistency
+    private fun debug_blob(blob: String) {
+        var metadataList = mutableListOf<KotlinKonanMetadata>()
+        KotlinKonanMetadataUtils.parseMetadata(blob, metadataList)
+
+        val body = metadataList.first().body
+        val gzipInputStream = java.util.zip.GZIPInputStream(ByteArrayInputStream(body))
+        val content = JsProtoBuf.Library.parseFrom(gzipInputStream)
+        gzipInputStream.close()
+
+        println(content.toString())
+        println(blob)
+    }
+
+    private fun serializeModule(moduleDescriptor: ModuleDescriptor): String {
+        val description = JsModuleDescriptor(
+                name = "foo",
+                kind = ModuleKind.PLAIN,
+                imported = listOf(),
+                data = moduleDescriptor
+             )
+
+        // TODO: eliminate this dependency on JavaScript compiler
+        val blobString =  KotlinJavascriptSerializationUtil.metadataAsString(description)
+        if (false) {
+            debug_blob(blobString)
+        }
+        return blobString
+
+    }
+
+    internal fun endModule(module: IrModuleFragment) {
+        val moduleAsString = serializeModule(module.descriptor)
+        val stringNode = metadataString(moduleAsString)
+        emitModuleMetadata("kmetadata", stringNode)
+    }
 }
 
 
+

backend.native/tests/build.gradle

@@ -12,18 +12,20 @@ abstract class KonanTest extends DefaultTask {
     def runtimeProject = project.project(":runtime")
     def llvmLlc = llvmTool("llc")
     def runtimeBc = new File("${runtimeProject.buildDir.canonicalPath}/runtime.bc")
+    def stdlibKtBc = new File("${runtimeProject.buildDir.canonicalPath}/stdlib.kt.bc")
     def mainC = 'main.c'
     String goldValue = null
     String testData = null
 
     public KonanTest(){
         dependsOn([project.project(":runtime").tasks['build'],
-                   project.parent.tasks['build']])
+                   project.parent.tasks['build'],
+                   project.project(":backend.native").tasks['stdlib']])
     }
 
 
 
-    abstract void compileTest(File sourceS, File runtimeS, File out)
+    abstract void compileTest(File sourceS, File runtimeS, File libraryPath, File out)
 
     private  File kt2bc(String ktSource) {
         def sourceKt = project.file(ktSource)
@@ -35,9 +37,9 @@ abstract class KonanTest extends DefaultTask {
             classpath = project.configurations.cli_bc
             jvmArgs "-ea"
             args "-output", "${sourceBc.absolutePath}",
-                    "-runtime", "${runtimeBc.absolutePath}",
+                 "-runtime", "${runtimeBc.absolutePath}",
-                    "-headers", project.project(':runtime').file('src/main/kotlin'),
+                 "-library", "${stdlibKtBc.absolutePath}",
-                    "${sourceKt.absolutePath}"
+                 "${sourceKt.absolutePath}"
 
             String libraryPath = "${project.llvmDir}/lib:${backendNative.buildDir.canonicalPath}/nativelibs"
             environment 'LD_LIBRARY_PATH'   : libraryPath
@@ -50,7 +52,8 @@ abstract class KonanTest extends DefaultTask {
     void executeTest() {
         def sourceS = bc2s(kt2bc(source))
         def exe = new File(sourceS.absolutePath.replace(".kt.S", ""))
-        compileTest(sourceS, bc2s(runtimeBc), exe)
+        def libraryPath = new File("${runtimeProject.file('src/main/bc').absolutePath}")
+        compileTest(sourceS, bc2s(runtimeBc), stdlibKtBc, exe)
         println "execution :${exe.absolutePath}"
 
         def out = null
@@ -116,24 +119,89 @@ abstract class KonanTest extends DefaultTask {
 }
 
 class UnitKonanTest extends KonanTest {
-    void compileTest(File sourceS, File runtimeS, File exe) {
+    void compileTest(File sourceS, File runtimeS, File stdlibKtBc, File exe) {
         def testC = sourceS.absolutePath.replace(".kt.S", "-test.c")
         project.execClang {
-            commandLine "clang", "${testC}", "${runtimeS.absolutePath}", "${sourceS.absolutePath}", "${mainC}",
+            commandLine "clang", "${testC}", "${runtimeS.absolutePath}", "${stdlibKtBc.absolutePath}", "${sourceS.absolutePath}", "${mainC}",
                     "-o", "${exe.absolutePath}", linkDl(), linkM(), rdynamic()
         }
     }
 }
 
 class RunKonanTest extends KonanTest {
-    void compileTest(File sourceS, File runtimeS, File exe) {
+    void compileTest(File sourceS, File runtimeS, File libraryPath, File exe) {
         project.execClang {
-            commandLine "clang", "-DRUN_TEST", "${runtimeS.absolutePath}", "${sourceS.absolutePath}", "${mainC}",
+            commandLine "clang", "-DRUN_TEST", "${runtimeS.absolutePath}", "${stdlibKtBc.absolutePath}", "${sourceS.absolutePath}", "${mainC}",
                     "-o", "${exe.absolutePath}", linkDl(), linkM(), rdynamic()
         }
     }
 }
 
+class LinkKonanTest extends KonanTest {
+    protected String lib
+
+    private File dir2bc(String ktSources) {
+        def sourcesKt = project.file(ktSources)
+        def libBc = new File("${sourcesKt.absolutePath}/bc/libout.kt.bc")
+        def libPath = "${libBc.absolutePath}"
+
+        project.javaexec {
+            main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt'
+            classpath = project.configurations.cli_bc
+            jvmArgs "-ea"
+            args "-output", "${libPath}",
+                 "-runtime", "${runtimeBc.absolutePath}",
+                 sourcesKt
+
+            String libraryPath = "${project.llvmDir}/lib:${backendNative.buildDir.canonicalPath}/nativelibs"
+            environment 'LD_LIBRARY_PATH'   : libraryPath
+            environment 'DYLD_LIBRARY_PATH' : libraryPath
+
+        }
+        return libBc
+    }
+
+    private File link(String ktSource, File libBc) {
+        def sourceKt = project.file(ktSource)
+        def outPath = "${sourceKt.absolutePath.replace('.kt', '.kt.bc')}"
+        def outBc = new File(outPath)
+
+        project.javaexec {
+            main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt'
+            classpath = project.configurations.cli_bc
+            jvmArgs "-ea"
+            args "-output", outPath,
+                 "-runtime", "${runtimeBc.absolutePath}",
+                 "-library", "${libBc.absolutePath}",
+                 "${sourceKt.absolutePath}"
+
+            String libraryPath = "${project.llvmDir}/lib:${backendNative.buildDir.canonicalPath}/nativelibs"
+            environment 'LD_LIBRARY_PATH'   : libraryPath
+            environment 'DYLD_LIBRARY_PATH' : libraryPath
+
+        }
+        return outBc
+    }
+
+    void compileTest(File sourceS, File runtimeS, File stdlibKtBc, File exe) {
+        def testC = sourceS.absolutePath.replace(".kt.S", "-test.c")
+        project.execClang {
+            commandLine "clang", "${testC}", "${runtimeS.absolutePath}", "${stdlibKtBc.absolutePath}", "${sourceS.absolutePath}", "${mainC}",
+                    "-o", "${exe.absolutePath}", linkDl(), rdynamic()
+        }
+    }
+
+
+    @TaskAction
+    void executeTest() {
+            def libBc = dir2bc(lib)
+            def outBc = link(source, libBc)
+    }
+
+
+
+}
+
 task run() {
     dependsOn(tasks.withType(KonanTest))
 }
@@ -289,3 +357,13 @@ task strdedup1(type: RunKonanTest) {
     goldValue = "true\ntrue\n"
     source = "datagen/literals/strdedup1.kt"
 }
+
+task intrinsic(type: UnitKonanTest) {
+    source = "codegen/function/intrinsic.kt"
+}
+
+task link(type: LinkKonanTest) {
+    source = "link/src/bar.kt"
+    lib = "link/lib"
+}
+

backend.native/tests/codegen/function/intrinsic-test.c

@@ -0,0 +1,10 @@
+extern void *resolve_symbol(const char*);
+
+int
+run_test() {
+  int (*intrinsic)(int) = resolve_symbol("kfun:intrinsic(Int)");
+
+  if (intrinsic(3) != 4) return 1;
+
+  return 0;
+}

backend.native/tests/codegen/function/intrinsic.kt

@@ -0,0 +1,11 @@
+
+// This code fails to link when bultins are taken from the 
+// frontend generated module, instead of our library.
+// Because the generated module doesn't see our name changing annotations,
+// the function names are all wrong.
+
+fun intrinsic(b: Int): Int {
+  var sum = 1
+  sum = sum + b
+  return sum
+}