diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt index 50f6b33198c..adb390a6814 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt @@ -263,7 +263,6 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) { // But we have to wait until the code generation phase, // to dump this information into generated file. var serializedLinkData: LinkData? = null - var serializedIr: ByteArray? = null var dataFlowGraph: ByteArray? = null @Deprecated("") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt index 2470e7a9644..15593914030 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt @@ -19,6 +19,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.languageVersionSettings import org.jetbrains.kotlin.ir.util.patchDeclarationParents +import org.jetbrains.kotlin.konan.library.impl.CombinedIrFileWriter import org.jetbrains.kotlin.psi2ir.Psi2IrConfiguration import org.jetbrains.kotlin.psi2ir.Psi2IrTranslator @@ -117,8 +118,8 @@ fun runTopLevelPhases(konanConfig: KonanConfig, environment: KotlinCoreEnvironme phaser.phase(KonanPhase.SERIALIZER) { val declarationTable = DeclarationTable(context.irModule!!.irBuiltins, DescriptorTable()) - val serializedIr = IrModuleSerializer(context, declarationTable, bodiesOnlyForInlines = context.config.isInteropStubs).serializedIrModule(context.irModule!!) - + val serializedIr = IrModuleSerializer( + context, declarationTable, bodiesOnlyForInlines = context.config.isInteropStubs).serializedIrModule(context.irModule!!) val serializer = KonanSerializationUtil(context, context.config.configuration.get(CommonConfigurationKeys.METADATA_VERSION)!!, declarationTable) context.serializedLinkData = serializer.serializeModule(context.moduleDescriptor, /*if (!context.config.isInteropStubs) serializedIr else null*/ serializedIr) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryWriter.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryWriter.kt index 73d332f72c1..28fbfd1eb30 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryWriter.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryWriter.kt @@ -33,7 +33,7 @@ class LinkData( class SerializedIr ( val module: ByteArray, - val declarations: Map, + val combinedDeclarationFilePath: String, val debugIndex: Map ) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/MetadataWriterImpl.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/MetadataWriterImpl.kt index 34f3401af04..1d224ce2457 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/MetadataWriterImpl.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/MetadataWriterImpl.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.backend.konan.library.impl import org.jetbrains.kotlin.backend.konan.library.LinkData +import org.jetbrains.kotlin.konan.file.File import org.jetbrains.kotlin.konan.library.KonanLibraryLayout internal class MetadataWriterImpl(libraryLayout: KonanLibraryLayout): KonanLibraryLayout by libraryLayout { @@ -25,13 +26,9 @@ internal class MetadataWriterImpl(libraryLayout: KonanLibraryLayout): KonanLibra packageFragmentFile(packageFqName, "${withLeadingZeros(i)}_$shortName").writeBytes(fragment) } } - linkData.ir?.declarations?.forEach { - val index = it.key.index.toULong().toString(16) - val file = if (it.key.isLocal) - hiddenDeclarationFile(index) - else - visibleDeclarationFile(index) - file.writeBytes(it.value) + linkData.ir?.combinedDeclarationFilePath?.let { + // TODO: use Files.move. + File(it).copyTo(irFile) } val lines = linkData.ir?.debugIndex?.map { entry -> "${entry.key}: ${entry.value}" } if (lines != null) irIndex.writeLines(lines) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrModuleSerializer.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrModuleSerializer.kt index b276d446ed6..43f075647f6 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrModuleSerializer.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrModuleSerializer.kt @@ -39,6 +39,8 @@ import org.jetbrains.kotlin.ir.expressions.impl.IrUnaryPrimitiveImpl import org.jetbrains.kotlin.ir.symbols.* import org.jetbrains.kotlin.ir.types.* import org.jetbrains.kotlin.ir.types.impl.IrStarProjectionImpl +import org.jetbrains.kotlin.konan.library.impl.CombinedIrFileWriter +import org.jetbrains.kotlin.konan.library.impl.DeclarationId import org.jetbrains.kotlin.metadata.KonanIr import org.jetbrains.kotlin.types.SimpleType import org.jetbrains.kotlin.types.StarProjectionImpl @@ -1050,30 +1052,35 @@ internal class IrModuleSerializer( .addAllLineStartOffsets(entry.lineStartOffsets.asIterable()) .build() - val topLevelDeclarations = mutableMapOf() - fun serializeIrFile(file: IrFile): KonanIr.IrFile { val proto = KonanIr.IrFile.newBuilder() .setFileEntry(serializeFileEntry(file.fileEntry)) .setFqName(serializeString(file.fqName.toString())) file.declarations.forEach { - if (it is IrTypeAlias) return@forEach - if (it.descriptor.isExpectMember && !it.descriptor.isSerializableExpectClass) { + if (it is IrTypeAlias || (it.descriptor.isExpectMember && !it.descriptor.isSerializableExpectClass)) { + writer.skipDeclaration() return@forEach } val byteArray = serializeDeclaration(it).toByteArray() val uniqId = declarationTable.uniqIdByDeclaration(it) - topLevelDeclarations.put(uniqId, byteArray) + writer.addDeclaration(DeclarationId(uniqId.index, uniqId.isLocal), byteArray) proto.addDeclarationId(protoUniqId(uniqId)) } return proto.build() } + lateinit var writer: CombinedIrFileWriter + fun serializeModule(module: IrModuleFragment): KonanIr.IrModule { val proto = KonanIr.IrModule.newBuilder() .setName(serializeString(module.name.toString())) + + val topLevelDeclarationsCount = module.files.sumBy { it.declarations.size } + + writer = CombinedIrFileWriter(topLevelDeclarationsCount) + module.files.forEach { proto.addFile(serializeIrFile(it)) } @@ -1095,6 +1102,6 @@ internal class IrModuleSerializer( fun serializedIrModule(module: IrModuleFragment): SerializedIr { val moduleHeader = serializeModule(module).toByteArray() - return SerializedIr(moduleHeader, topLevelDeclarations, declarationTable.debugIndex) + return SerializedIr(moduleHeader, writer.finishWriting().absolutePath, declarationTable.debugIndex) } } diff --git a/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanLibraryLayout.kt b/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanLibraryLayout.kt index c763922834b..a860f255c53 100644 --- a/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanLibraryLayout.kt +++ b/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanLibraryLayout.kt @@ -1,3 +1,19 @@ +/** + * Copyright 2010-2019 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.jetbrains.kotlin.konan.library import org.jetbrains.kotlin.konan.file.File @@ -46,10 +62,8 @@ interface KonanLibraryLayout { File(packageFragmentsDir(packageFqName), "$partName$KLIB_METADATA_FILE_EXTENSION_WITH_DOT") val irDir get() = File(libDir, "ir") - fun hiddenDeclarationFile(declarationId: String) - = File(irDir, "hidden_$declarationId.knd") - fun visibleDeclarationFile(declarationId: String) - = File(irDir, "visible_$declarationId.knd") + val irFile + get() = File(irDir, "irCombined.knd") val irHeader get() = File(irDir, "irHeaders.kni") val irIndex: File diff --git a/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanMetadataReader.kt b/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanMetadataReader.kt index 0961fbfb514..57ae74f6e12 100644 --- a/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanMetadataReader.kt +++ b/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanMetadataReader.kt @@ -3,6 +3,4 @@ package org.jetbrains.kotlin.konan.library interface MetadataReader { fun loadSerializedModule(libraryLayout: KonanLibraryLayout): ByteArray fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String, partName: String): ByteArray - fun loadIrHeader(libraryLayout: KonanLibraryLayout): ByteArray - fun loadIrDeclaraton(libraryLayout: KonanLibraryLayout, index: Long, isLocal: Boolean): ByteArray } diff --git a/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/CombinedIrFileAccessor.kt b/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/CombinedIrFileAccessor.kt new file mode 100644 index 00000000000..235f1312aa2 --- /dev/null +++ b/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/CombinedIrFileAccessor.kt @@ -0,0 +1,94 @@ +/** + * Copyright 2010-2019 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.konan.library.impl + +import org.jetbrains.kotlin.konan.file.File +import java.io.RandomAccessFile +import java.nio.channels.FileChannel + +data class DeclarationId(val id: Long, val isLocal: Boolean) + +class CombinedIrFileReader(file: File) { + private val buffer = file.map(FileChannel.MapMode.READ_ONLY) + private val declarationToOffsetSize = mutableMapOf>() + + init { + val declarationsCount = buffer.int + for (i in 0 until declarationsCount) { + val id = buffer.long + val isLocal = buffer.int != 0 + val offset = buffer.int + val size = buffer.int + declarationToOffsetSize[DeclarationId(id, isLocal)] = offset to size + } + } + + fun declarationBytes(id: DeclarationId): ByteArray { + val offsetSize = declarationToOffsetSize[id] ?: throw Error("No declaration with $id here") + val result = ByteArray(offsetSize.second) + buffer.position(offsetSize.first) + buffer.get(result, 0, offsetSize.second) + return result + } +} + +private const val SINGLE_INDEX_RECORD_SIZE = 20 // sizeof(Long) + 3 * sizeof(Int). +private const val INDEX_HEADER_SIZE = 4 // sizeof(Int). + +class CombinedIrFileWriter(val declarationCount: Int) { + private var currentDeclaration = 0 + private var currentPosition = 0 + private val file = org.jetbrains.kotlin.konan.file.createTempFile("ir").deleteOnExit() + private val randomAccessFile = RandomAccessFile(file.path, "rw") + + init { + randomAccessFile.writeInt(declarationCount) + assert(randomAccessFile.filePointer.toInt() == INDEX_HEADER_SIZE) + for (i in 0 until declarationCount) { + randomAccessFile.writeLong(-1) // id + randomAccessFile.writeInt(-1) // isLocal + randomAccessFile.writeInt(-1) // offset + randomAccessFile.writeInt(-1) // size + } + currentPosition = randomAccessFile.filePointer.toInt() + assert(currentPosition == INDEX_HEADER_SIZE + SINGLE_INDEX_RECORD_SIZE * declarationCount) + } + + fun skipDeclaration() { + currentDeclaration++ + } + + fun addDeclaration(id: DeclarationId, bytes: ByteArray) { + randomAccessFile.seek((currentDeclaration * SINGLE_INDEX_RECORD_SIZE + INDEX_HEADER_SIZE).toLong()) + randomAccessFile.writeLong(id.id) + randomAccessFile.writeInt(if (id.isLocal) 1 else 0) + randomAccessFile.writeInt(currentPosition) + randomAccessFile.writeInt(bytes.size) + randomAccessFile.seek(currentPosition.toLong()) + randomAccessFile.write(bytes) + assert(randomAccessFile.filePointer < Int.MAX_VALUE.toLong()) + currentPosition = randomAccessFile.filePointer.toInt() + currentDeclaration++ + } + + fun finishWriting(): File { + assert(currentDeclaration == declarationCount) + randomAccessFile.close() + return file + } +} + diff --git a/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/DefaultMetadataReaderImpl.kt b/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/DefaultMetadataReaderImpl.kt index 8e1ab774473..5b1eb21e50c 100644 --- a/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/DefaultMetadataReaderImpl.kt +++ b/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/DefaultMetadataReaderImpl.kt @@ -11,15 +11,4 @@ internal object DefaultMetadataReaderImpl : MetadataReader { override fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String, partName: String): ByteArray = libraryLayout.packageFragmentFile(fqName, partName).readBytes() - override fun loadIrHeader(libraryLayout: KonanLibraryLayout): ByteArray = - libraryLayout.irHeader.readBytes() - - override fun loadIrDeclaraton(libraryLayout: KonanLibraryLayout, index: Long, isLocal: Boolean): ByteArray { - val name = index.toULong().toString(16) - val file = if (isLocal) - libraryLayout.hiddenDeclarationFile(name) - else - libraryLayout.visibleDeclarationFile(name) - return file.readBytes() - } } diff --git a/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryImpl.kt b/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryImpl.kt index 446a5c154c5..77bf2096ab8 100644 --- a/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryImpl.kt +++ b/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryImpl.kt @@ -84,9 +84,22 @@ class KonanLibraryImpl( } } - override val irHeader: ByteArray? by lazy { layout.inPlace { library -> library.irHeader.let { if (it.exists) metadataReader.loadIrHeader(library) else null }}} + override val irHeader: ByteArray? by lazy { layout.inPlace { library -> library.irHeader.let { + if (it.exists) loadIrHeader() else null } + }} - override fun irDeclaration(index: Long, isLocal: Boolean) = layout.inPlace { metadataReader.loadIrDeclaraton(it, index, isLocal) } + override fun irDeclaration(index: Long, isLocal: Boolean) = loadIrDeclaraton(index, isLocal) override fun toString() = "$libraryName[default=$isDefault]" + + private val combinedDeclarations: CombinedIrFileReader by lazy { + CombinedIrFileReader(layout.realFiles { + it.irFile + }) + } + private fun loadIrHeader(): ByteArray = + layout.inPlace { it.irHeader.readBytes() } + + private fun loadIrDeclaraton(index: Long, isLocal: Boolean) = + combinedDeclarations.declarationBytes(DeclarationId(index, isLocal)) } diff --git a/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryLayoutImpl.kt b/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryLayoutImpl.kt index cdb65b14536..def00ea7053 100644 --- a/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryLayoutImpl.kt +++ b/shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryLayoutImpl.kt @@ -19,7 +19,7 @@ private class ZippedKonanLibraryLayout(val klibFile: File, override val target: override val libraryName = klibFile.path.removeSuffixIfPresent(KLIB_FILE_EXTENSION_WITH_DOT) - override val libDir: File= File("/") + override val libDir = File("/") override fun realFiles(action: (KonanLibraryLayout) -> T): T { return action(FileExtractor(this))!! @@ -70,6 +70,8 @@ private class FileExtractor(val zippedLibraryLayout: ZippedKonanLibraryLayout): override val linkdataDir: File by lazy { extractDir(super.linkdataDir) } + override val irFile: File by lazy { extract(super.irFile) } + fun extract(file: File): File = zippedLibraryLayout.klibFile.withZipFileSystem { zipFileSystem -> val temporary = createTempFile(file.name) zipFileSystem.file(file).copyTo(temporary) diff --git a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/KonanAbiVersion.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/KonanAbiVersion.kt index 23e51c2e656..3a96f0fc6c5 100644 --- a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/KonanAbiVersion.kt +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/KonanAbiVersion.kt @@ -1,3 +1,19 @@ +/** + * Copyright 2010-2019 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.jetbrains.kotlin.konan fun String.parseKonanAbiVersion(): KonanAbiVersion { @@ -6,7 +22,7 @@ fun String.parseKonanAbiVersion(): KonanAbiVersion { data class KonanAbiVersion(val version: Int) { companion object { - val CURRENT = KonanAbiVersion(6) + val CURRENT = KonanAbiVersion(7) } override fun toString() = "$version" } diff --git a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/File.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/File.kt index 66431182694..d4119212479 100644 --- a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/File.kt +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/File.kt @@ -22,7 +22,10 @@ import org.jetbrains.kotlin.konan.util.removeSuffixIfPresent import java.io.BufferedReader import java.io.InputStream import java.io.InputStreamReader +import java.io.RandomAccessFile import java.net.URI +import java.nio.MappedByteBuffer +import java.nio.channels.FileChannel import java.nio.file.* import java.nio.file.attribute.BasicFileAttributes @@ -108,8 +111,17 @@ data class File constructor(internal val javaPath: Path) { return FileVisitResult.CONTINUE } }) - } + + fun map(mode: FileChannel.MapMode = FileChannel.MapMode.READ_ONLY, + start: Long = 0, size: Long = -1): MappedByteBuffer { + val file = RandomAccessFile(path, + if (mode == FileChannel.MapMode.READ_ONLY) "r" else "rw") + val fileSize = if (mode == FileChannel.MapMode.READ_ONLY) + file.length() else size.also { assert(size != -1L) } + return file.channel.map(mode, start, fileSize) // Shall we .also { file.close() }? + } + fun deleteOnExit(): File { // Works only on the default file system, // but that's okay for now.