azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Here goes stdlib separation.

Browse Source 
$ gradlew backend.native:stdlib

builds you stdlib.kt.bc now.

The test runs are taught to supply
-library stdlib.kt.bc to kotlin compiler for proper imports resolution,
and then later stdlib.kt.bc is provided to clang for the final link step.
This commit is contained in:
Alexander Gorshenev
2016-11-17 17:37:43 +03:00
committed by alexander-gorshenev
parent b0d10e45d8
commit 1d265bc50c
11 changed files with 294 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend.native/build.gradle
+20 -1
View File
@@ -161,19 +161,38 @@ dependencies {
build.dependsOn 'compilerClasses','cli_bcClasses','bc_frontendClasses'
task stdlib(type: JavaExec) {
main 'org.jetbrains.kotlin.cli.bc.K2NativeKt'
classpath configurations.cli_bc
args project(':runtime').file('src/main/kotlin')
args '-output', project(':runtime').file('build/stdlib.kt.bc')
jvmArgs '-ea', "-Djava.library.path=${project.buildDir}/nativelibs"
dependsOn ':runtime:build'
doFirst {
args '-runtime', project(':runtime').build.outputs.files.singleFile
}
environment 'LD_LIBRARY_PATH' : "${llvmDir}/lib:${project.buildDir}/nativelibs"
}
task run(type: JavaExec) {
main 'org.jetbrains.kotlin.cli.bc.K2NativeKt'
classpath configurations.cli_bc
args 'tests/codegen/function/sum.kt'
args '-headers', project(':runtime').file('src/main/kotlin')
args '-library', project(':runtime').file('build/stdlib.kt.bc')
args '-output', "$buildDir/out.bc"
jvmArgs '-ea', "-Djava.library.path=${project.buildDir}/nativelibs"
dependsOn ':runtime:build'
dependsOn 'stdlib'
doFirst {
args '-runtime', project(':runtime').build.outputs.files.singleFile
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt
+5 -11
View File
@@ -37,25 +37,19 @@ class NativeAnalyzer(
class K2Native : CLICompiler<K2NativeCompilerArguments>() {
val defaultModuleName = "main";
override fun doExecute(arguments : K2NativeCompilerArguments,
configuration : CompilerConfiguration,
rootDisposable: Disposable
): ExitCode {
configuration.put(CommonConfigurationKeys.MODULE_NAME, defaultModuleName )
configuration.put(CommonConfigurationKeys.MODULE_NAME, defaultModuleName)
configuration.addKotlinSourceRoots(arguments.freeArgs)
// TODO: remove, once linker is available.
if (arguments.headers != null) {
configuration.addKotlinSourceRoots(arguments.headers.asList())
}
// As of now we don't have any library mechnanism
var libraryFiles = listOf<String>()
configuration.put(KonanConfigurationKeys.LIBRARY_FILES, libraryFiles)
val libraries = arguments.libraries?.asList<String>() ?: listOf<String>()
configuration.put(KonanConfigurationKeys.LIBRARY_FILES, libraries)
libraries.forEach{println(it)}
val environment = KotlinCoreEnvironment.createForProduction(rootDisposable,
configuration, Arrays.asList<String>("extensions/common.xml"))
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.java
+4 -3
View File
@@ -14,9 +14,10 @@ public class K2NativeCompilerArguments extends CommonCompilerArguments {
@ValueDescription("<path>")
public String runtimeFile;
@Argument(value = "headers", description = "Header files used for compilation")
@Argument(value = "library", description = "Bitcode file with metadata attached")
@ValueDescription("<path>")
public String[] headers;
public String[] libraries;
public K2NativeCompilerArguments() {
}
@@ -24,7 +25,7 @@ public class K2NativeCompilerArguments extends CommonCompilerArguments {
public K2NativeCompilerArguments(K2NativeCompilerArguments arguments) {
this.outputFile = arguments.outputFile;
this.runtimeFile = arguments.runtimeFile;
this.headers = arguments.headers;
this.libraries = arguments.libraries;
}
public CommonCompilerArguments copy() {
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/KonanConfig.kt
+12 -26
View File
@@ -23,14 +23,9 @@ import org.jetbrains.kotlin.backend.konan.llvm.KotlinKonanMetadataUtils
class KonanConfig(val project: Project, val configuration: CompilerConfiguration) {
private val storageManager = LockBasedStorageManager()
private val sourceFilesFromLibraries = listOf<KtFile>()
private var metadata = listOf<KotlinKonanMetadata>()
internal var moduleDescriptors: MutableList<JsModuleDescriptor<ModuleDescriptorImpl>>? = null
private val libraries = configuration.getList(KonanConfigurationKeys.LIBRARY_FILES)
init {
val libraries = configuration.getList(KonanConfigurationKeys.LIBRARY_FILES)
metadata = KotlinKonanMetadataUtils.loadLibMetadata(libraries)
}
private val metadata = KotlinKonanMetadataUtils.loadLibMetadata(libraries)
val moduleId: String
get() = configuration.getNotNull(CommonConfigurationKeys.MODULE_NAME)
@@ -38,26 +33,25 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
val moduleKind: ModuleKind
get() = configuration.get(KonanConfigurationKeys.MODULE_KIND)!!
fun getModuleDescriptors(): MutableList<JsModuleDescriptor<ModuleDescriptorImpl>> {
if (moduleDescriptors != null) return moduleDescriptors!!
// We reuse JsModuleDescriptor for serialization for now, as we haven't got one for Konan yet.
internal val moduleDescriptors: MutableList<JsModuleDescriptor<ModuleDescriptorImpl>> by lazy {
val jsDescriptors = mutableListOf<JsModuleDescriptor<ModuleDescriptorImpl>>()
val descriptors = mutableListOf<ModuleDescriptorImpl>()
moduleDescriptors = mutableListOf<JsModuleDescriptor<ModuleDescriptorImpl>>()
val kotlinModuleDescriptors = mutableListOf<ModuleDescriptorImpl>()
for (metadataEntry in metadata) {
val descriptor = createModuleDescriptor(metadataEntry)
moduleDescriptors!!.add(descriptor)
kotlinModuleDescriptors.add(descriptor.data)
jsDescriptors.add(descriptor)
descriptors.add(descriptor.data)
}
for (module in moduleDescriptors!!) {
setDependencies(module.data, kotlinModuleDescriptors)
for (module in jsDescriptors) {
setDependencies(module.data, descriptors)
}
return moduleDescriptors!!
jsDescriptors
}
// We reuse JsModuleDescriptor for serialization for now, as we haven't got one for Konan yet.
private fun createModuleDescriptor(metadata: KotlinKonanMetadata): JsModuleDescriptor<ModuleDescriptorImpl> {
val moduleDescriptor = ModuleDescriptorImpl(
@@ -77,14 +71,6 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
private fun setDependencies(module: ModuleDescriptorImpl, modules: List<ModuleDescriptorImpl>) {
module.setDependencies(modules.plus(KonanPlatform.builtIns.builtInsModule))
}
fun withJsLibAdded(files: Collection<KtFile>, config: KonanConfig): Collection<KtFile> {
val allFiles = mutableListOf<KtFile>()
allFiles.addAll(files)
// We don't store source files in the bitcode for Konan, dont't we?
//allFiles.addAll(config.getSourceFilesFromLibraries());
return allFiles
}
}
}
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/TopDownAnalyzerFacadeForKonan.kt
+5 -7
View File
@@ -38,7 +38,7 @@ object TopDownAnalyzerFacadeForKonan {
// Make sure the compiler produced BuiltIns module comes in last
context.setDependencies(
listOf(context.module) +
config.getModuleDescriptors().map { it.data } +
config.moduleDescriptors.map { it.data } +
listOf(compilerBuiltInsModule)
)
return analyzeFilesWithGivenTrace(files, BindingTraceContext(), context, config)
@@ -51,14 +51,12 @@ object TopDownAnalyzerFacadeForKonan {
config: KonanConfig
): AnalysisResult {
val allFiles = KonanConfig.withJsLibAdded(files, config)
// we print out each file we compile for now
allFiles.forEach{println(it)}
files.forEach{println(it)}
val analyzerForKonan = createTopDownAnalyzerForKonan(
moduleContext, trace,
FileBasedDeclarationProviderFactory(moduleContext.storageManager, allFiles),
FileBasedDeclarationProviderFactory(moduleContext.storageManager, files),
config.configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS, LanguageVersionSettingsImpl.DEFAULT)
)
@@ -66,9 +64,9 @@ object TopDownAnalyzerFacadeForKonan {
return AnalysisResult.success(trace.bindingContext, moduleContext.module)
}
fun checkForErrors(allFiles: Collection<KtFile>, bindingContext: BindingContext) {
fun checkForErrors(files: Collection<KtFile>, bindingContext: BindingContext) {
AnalyzingUtils.throwExceptionOnErrors(bindingContext)
for (file in allFiles) {
for (file in files) {
AnalyzingUtils.checkForSyntacticErrors(file)
}
}
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt
+9
View File
@@ -85,6 +85,13 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
element.acceptChildrenVoid(this)
}
//-------------------------------------------------------------------------//
override fun visitModuleFragment(module: IrModuleFragment) {
logger.log("visitModule : ${ir2string(module)}")
module.acceptChildrenVoid(this)
metadator.endModule(module)
}
//-------------------------------------------------------------------------//
override fun visitWhen(expression: IrWhen) {
@@ -183,6 +190,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
//-------------------------------------------------------------------------//
override fun visitFunction(declaration: IrFunction) {
logger.log("visitFunction : ${ir2string(declaration)}")
codegen.function(declaration)
metadator.function(declaration)
declaration.acceptChildrenVoid(this)
@@ -191,6 +199,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
//-------------------------------------------------------------------------//
override fun visitClass(declaration: IrClass) {
logger.log("visitClass : ${ir2string(declaration)}")
if (declaration.descriptor.kind == ClassKind.ANNOTATION_CLASS) {
// do not generate any code for annotation classes as a workaround for NotImplementedError
return
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/KotlinKonanMetadataUtils.kt
+3 -4
View File
@@ -60,10 +60,8 @@ object KotlinKonanMetadataUtils {
fun loadMetadata(file: File): List<KotlinKonanMetadata> {
assert(file.exists()) { "Library $file not found" }
val metadataList = arrayListOf<KotlinKonanMetadata>()
// For the time being we don't have a reader, so just produce nothing
//val content = readModuleMetadata(file)
//parseMetadata(content, metadataList)
val content = readModuleMetadata(file)
parseMetadata(content, metadataList)
return metadataList
}
@@ -115,3 +113,4 @@ object KotlinKonanMetadataUtils {
return allMetadata;
}
}
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/MetadataGenerator.kt
+127
View File
@@ -4,9 +4,98 @@ package org.jetbrains.kotlin.backend.konan.llvm
import kotlin_native.interop.*
import llvm.*
import org.jetbrains.kotlin.ir.declarations.IrFunction
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.backend.konan.llvm.BinaryLinkdata.*
import org.jetbrains.kotlin.types.KotlinType
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.serialization.deserialization.DeserializationConfiguration
import org.jetbrains.kotlin.serialization.js.*
import org.jetbrains.kotlin.utils.*
import java.io.*
fun readModuleMetadata(file: File): String {
val reader = MetadataReader(file)
var metadataString: String? = null
try {
val metadataNode = reader.namedMetadataNode("kmetadata", 0);
val stringNode = reader.metadataOperand(metadataNode, 0)!!
metadataString = reader.string(stringNode)
} finally {
reader.close()
}
return metadataString!!
}
class MetadataReader(file: File) {
lateinit var llvmModule: LLVMOpaqueModule
lateinit var llvmContext: LLVMOpaqueContext
init {
memScoped {
val bufRef = alloc(LLVMOpaqueMemoryBuffer.ref)
val errorRef = alloc(Int8Box.ref)
val res = LLVMCreateMemoryBufferWithContentsOfFile(file.toString(), bufRef, errorRef)
if (res != 0) {
throw Error(errorRef.value?.asCString()?.toString())
}
llvmContext = LLVMContextCreate()!!
val moduleRef = alloc(LLVMOpaqueModule.ref)
val parseResult = LLVMParseBitcodeInContext2(llvmContext, bufRef.value, moduleRef)
if (parseResult != 0) {
throw Error(parseResult.toString())
}
llvmModule = moduleRef.value!!
}
}
fun string(node: LLVMOpaqueValue): String {
memScoped {
val len = alloc(Int32Box)
val str1 = LLVMGetMDString(node, len)!!
val str = str1.asCString().toString()
return str
}
}
fun namedMetadataNode(name: String, index: Int): LLVMOpaqueValue {
memScoped {
val nodeCount = LLVMGetNamedMetadataNumOperands(llvmModule, "kmetadata")!!
val nodeArray = alloc(array[nodeCount](Ref to LLVMOpaqueValue))
LLVMGetNamedMetadataOperands(llvmModule, "kmetadata", nodeArray[0])!!
return nodeArray[0].value!!
}
}
fun metadataOperand(metadataNode: LLVMOpaqueValue, index: Int): LLVMOpaqueValue {
memScoped {
val operandCount = LLVMGetMDNodeNumOperands(metadataNode)!!
val operandArray = alloc(array[operandCount](Ref to LLVMOpaqueValue))
LLVMGetMDNodeOperands(metadataNode, operandArray[0])!!
return operandArray[0].value!!
}
}
fun close() {
LLVMDisposeModule(llvmModule)
LLVMContextDispose(llvmContext)
}
}
internal class MetadataGenerator(override val context: Context): ContextUtils {
private fun metadataString(str: String): LLVMOpaqueValue {
@@ -74,6 +163,44 @@ internal class MetadataGenerator(override val context: Context): ContextUtils {
val md = metadataFun(fn, proto)
emitModuleMetadata("kfun", md);
}
// Quick check consistency
private fun debug_blob(blob: String) {
var metadataList = mutableListOf<KotlinKonanMetadata>()
KotlinKonanMetadataUtils.parseMetadata(blob, metadataList)
val body = metadataList.first().body
val gzipInputStream = java.util.zip.GZIPInputStream(ByteArrayInputStream(body))
val content = JsProtoBuf.Library.parseFrom(gzipInputStream)
gzipInputStream.close()
println(content.toString())
println(blob)
}
private fun serializeModule(moduleDescriptor: ModuleDescriptor): String {
val description = JsModuleDescriptor(
name = "foo",
kind = ModuleKind.PLAIN,
imported = listOf(),
data = moduleDescriptor
)
// TODO: eliminate this dependency on JavaScript compiler
val blobString = KotlinJavascriptSerializationUtil.metadataAsString(description)
if (false) {
debug_blob(blobString)
}
return blobString
}
internal fun endModule(module: IrModuleFragment) {
val moduleAsString = serializeModule(module.descriptor)
val stringNode = metadataString(moduleAsString)
emitModuleMetadata("kmetadata", stringNode)
}
}
backend.native/tests/build.gradle
+88 -10
View File
@@ -12,18 +12,20 @@ abstract class KonanTest extends DefaultTask {
def runtimeProject = project.project(":runtime")
def llvmLlc = llvmTool("llc")
def runtimeBc = new File("${runtimeProject.buildDir.canonicalPath}/runtime.bc")
def stdlibKtBc = new File("${runtimeProject.buildDir.canonicalPath}/stdlib.kt.bc")
def mainC = 'main.c'
String goldValue = null
String testData = null
public KonanTest(){
dependsOn([project.project(":runtime").tasks['build'],
project.parent.tasks['build']])
project.parent.tasks['build'],
project.project(":backend.native").tasks['stdlib']])
}
abstract void compileTest(File sourceS, File runtimeS, File out)
abstract void compileTest(File sourceS, File runtimeS, File libraryPath, File out)
private File kt2bc(String ktSource) {
def sourceKt = project.file(ktSource)
@@ -35,9 +37,9 @@ abstract class KonanTest extends DefaultTask {
classpath = project.configurations.cli_bc
jvmArgs "-ea"
args "-output", "${sourceBc.absolutePath}",
"-runtime", "${runtimeBc.absolutePath}",
"-headers", project.project(':runtime').file('src/main/kotlin'),
"${sourceKt.absolutePath}"
"-runtime", "${runtimeBc.absolutePath}",
"-library", "${stdlibKtBc.absolutePath}",
"${sourceKt.absolutePath}"
String libraryPath = "${project.llvmDir}/lib:${backendNative.buildDir.canonicalPath}/nativelibs"
environment 'LD_LIBRARY_PATH' : libraryPath
@@ -50,7 +52,8 @@ abstract class KonanTest extends DefaultTask {
void executeTest() {
def sourceS = bc2s(kt2bc(source))
def exe = new File(sourceS.absolutePath.replace(".kt.S", ""))
compileTest(sourceS, bc2s(runtimeBc), exe)
def libraryPath = new File("${runtimeProject.file('src/main/bc').absolutePath}")
compileTest(sourceS, bc2s(runtimeBc), stdlibKtBc, exe)
println "execution :${exe.absolutePath}"
def out = null
@@ -116,24 +119,89 @@ abstract class KonanTest extends DefaultTask {
}
class UnitKonanTest extends KonanTest {
void compileTest(File sourceS, File runtimeS, File exe) {
void compileTest(File sourceS, File runtimeS, File stdlibKtBc, File exe) {
def testC = sourceS.absolutePath.replace(".kt.S", "-test.c")
project.execClang {
commandLine "clang", "${testC}", "${runtimeS.absolutePath}", "${sourceS.absolutePath}", "${mainC}",
commandLine "clang", "${testC}", "${runtimeS.absolutePath}", "${stdlibKtBc.absolutePath}", "${sourceS.absolutePath}", "${mainC}",
"-o", "${exe.absolutePath}", linkDl(), linkM(), rdynamic()
}
}
}
class RunKonanTest extends KonanTest {
void compileTest(File sourceS, File runtimeS, File exe) {
void compileTest(File sourceS, File runtimeS, File libraryPath, File exe) {
project.execClang {
commandLine "clang", "-DRUN_TEST", "${runtimeS.absolutePath}", "${sourceS.absolutePath}", "${mainC}",
commandLine "clang", "-DRUN_TEST", "${runtimeS.absolutePath}", "${stdlibKtBc.absolutePath}", "${sourceS.absolutePath}", "${mainC}",
"-o", "${exe.absolutePath}", linkDl(), linkM(), rdynamic()
}
}
}
class LinkKonanTest extends KonanTest {
protected String lib
private File dir2bc(String ktSources) {
def sourcesKt = project.file(ktSources)
def libBc = new File("${sourcesKt.absolutePath}/bc/libout.kt.bc")
def libPath = "${libBc.absolutePath}"
project.javaexec {
main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt'
classpath = project.configurations.cli_bc
jvmArgs "-ea"
args "-output", "${libPath}",
"-runtime", "${runtimeBc.absolutePath}",
sourcesKt
String libraryPath = "${project.llvmDir}/lib:${backendNative.buildDir.canonicalPath}/nativelibs"
environment 'LD_LIBRARY_PATH' : libraryPath
environment 'DYLD_LIBRARY_PATH' : libraryPath
}
return libBc
}
private File link(String ktSource, File libBc) {
def sourceKt = project.file(ktSource)
def outPath = "${sourceKt.absolutePath.replace('.kt', '.kt.bc')}"
def outBc = new File(outPath)
project.javaexec {
main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt'
classpath = project.configurations.cli_bc
jvmArgs "-ea"
args "-output", outPath,
"-runtime", "${runtimeBc.absolutePath}",
"-library", "${libBc.absolutePath}",
"${sourceKt.absolutePath}"
String libraryPath = "${project.llvmDir}/lib:${backendNative.buildDir.canonicalPath}/nativelibs"
environment 'LD_LIBRARY_PATH' : libraryPath
environment 'DYLD_LIBRARY_PATH' : libraryPath
}
return outBc
}
void compileTest(File sourceS, File runtimeS, File stdlibKtBc, File exe) {
def testC = sourceS.absolutePath.replace(".kt.S", "-test.c")
project.execClang {
commandLine "clang", "${testC}", "${runtimeS.absolutePath}", "${stdlibKtBc.absolutePath}", "${sourceS.absolutePath}", "${mainC}",
"-o", "${exe.absolutePath}", linkDl(), rdynamic()
}
}
@TaskAction
void executeTest() {
def libBc = dir2bc(lib)
def outBc = link(source, libBc)
}
}
task run() {
dependsOn(tasks.withType(KonanTest))
}
@@ -289,3 +357,13 @@ task strdedup1(type: RunKonanTest) {
goldValue = "true\ntrue\n"
source = "datagen/literals/strdedup1.kt"
}
task intrinsic(type: UnitKonanTest) {
source = "codegen/function/intrinsic.kt"
}
task link(type: LinkKonanTest) {
source = "link/src/bar.kt"
lib = "link/lib"
}
backend.native/tests/codegen/function/intrinsic-test.c
+10
View File
@@ -0,0 +1,10 @@
extern void *resolve_symbol(const char*);
int
run_test() {
int (*intrinsic)(int) = resolve_symbol("kfun:intrinsic(Int)");
if (intrinsic(3) != 4) return 1;
return 0;
}
backend.native/tests/codegen/function/intrinsic.kt
+11
View File
@@ -0,0 +1,11 @@
// This code fails to link when bultins are taken from the
// frontend generated module, instead of our library.
// Because the generated module doesn't see our name changing annotations,
// the function names are all wrong.
fun intrinsic(b: Int): Int {
var sum = 1
sum = sum + b
return sum
}