Separated Zipped and Unzipped libraries.

Picking up files right from the zip filesystem, if possible.
This commit is contained in:
Alexander Gorshenev
2017-07-03 12:54:08 +03:00
committed by alexander-gorshenev
parent 56c1f2b07c
commit 5b25613e23
11 changed files with 293 additions and 151 deletions
@@ -18,15 +18,15 @@ package org.jetbrains.kotlin.backend.konan
import com.intellij.openapi.project.Project
import org.jetbrains.kotlin.backend.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.backend.konan.library.impl.LibraryReaderImpl
import org.jetbrains.kotlin.backend.konan.library.KonanLibrarySearchPathResolver
import org.jetbrains.kotlin.backend.konan.library.impl.LibraryReaderImpl
import org.jetbrains.kotlin.backend.konan.util.File
import org.jetbrains.kotlin.backend.konan.util.profile
import org.jetbrains.kotlin.backend.konan.util.suffixIfNot
import org.jetbrains.kotlin.backend.konan.util.removeSuffixIfPresent
import org.jetbrains.kotlin.backend.konan.util.suffixIfNot
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.backend.konan.util.File
import org.jetbrains.kotlin.konan.target.*
import org.jetbrains.kotlin.konan.target.TargetManager.*
import org.jetbrains.kotlin.konan.target.CompilerOutputKind.*
@@ -203,7 +203,7 @@ internal class LinkStage(val context: Context) {
val libraries = context.config.libraries
fun llvmLto(files: List<BitcodeFile>): ObjectFile {
val tmpCombined = File.createTempFile("combined", ".o")
val tmpCombined = createTempFile("combined", ".o")
tmpCombined.deleteOnExit()
val combined = tmpCombined.absolutePath
@@ -20,13 +20,11 @@ import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.backend.konan.util.File
// This scheme describes the Konan Library (klib) layout.
interface KonanLibrary {
interface KonanLibraryLayout {
val libDir: File
val target: KonanTarget?
// This is a default implementation. Can't make it an assignment.
get() = null
val klibFile
get() = File("${libDir.path}.klib")
val manifestFile
get() = File(libDir, "manifest")
val resourcesDir
@@ -47,3 +45,7 @@ interface KonanLibrary {
= File(linkdataDir, if (packageName == "") "root_package" else "package_$packageName")
}
interface KonanLibrary: KonanLibraryLayout {
val libraryName: String
}
@@ -50,8 +50,7 @@ class KonanLibrarySearchPathResolver(repositories: List<String>,
val noSuffix = File(candidate.path.removeSuffixIfPresent(".klib"))
val withSuffix = File(candidate.path.suffixIfNot(".klib"))
if (withSuffix.exists) {
// We always returing the name without suffix here.
return noSuffix
return withSuffix
}
if (noSuffix.exists) {
return noSuffix
@@ -0,0 +1,111 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.backend.konan.library.impl
import org.jetbrains.kotlin.backend.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.util.*
import org.jetbrains.kotlin.konan.target.KonanTarget
open class ZippedKonanLibrary(val klibFile: File, override val target: KonanTarget? = null): KonanLibrary {
init {
if (!klibFile.exists) {
error("Could not find $klibFile.")
}
if (!klibFile.isFile) {
error("Expected $klibFile to be a regular file.")
}
}
override val libraryName = klibFile.path.removeSuffixIfPresent(".klib")
override val libDir by lazy {
klibFile.asZipRoot
}
fun unpackTo(newDir: File) {
if (newDir.exists) {
if (newDir.isDirectory)
newDir.deleteRecursively()
else
newDir.delete()
}
libDir.recursiveCopyTo(newDir)
if (!newDir.exists) error("Could not unpack $klibFile as $newDir.")
}
}
// This class automatically extracts pieces of
// the library on first access. Use it if you need
// to pass extracted files to an external tool.
// Otherwise, stick to ZippedKonanLibrary.
class FileExtractor(zippedLibrary: KonanLibrary): KonanLibrary by zippedLibrary {
override val manifestFile: File by lazy {
extract(super.manifestFile)
}
override val resourcesDir: File by lazy {
extractDir(super.resourcesDir)
}
override val kotlinDir: File by lazy {
extractDir(super.kotlinDir)
}
override val nativeDir: File by lazy {
extractDir(super.nativeDir)
}
override val linkdataDir: File by lazy {
extractDir(super.linkdataDir)
}
fun extract(file: File): File {
val temporary = createTempFile(file.name)
file.copyTo(temporary)
temporary.deleteOnExit()
return temporary
}
fun extractDir(directory: File): File {
val temporary = createTempDir(directory.name)
directory.recursiveCopyTo(temporary)
temporary.deleteOnExitRecursively()
return temporary
}
}
class UnzippedKonanLibrary(override val libDir: File, override val target: KonanTarget? = null): KonanLibrary {
override val libraryName = libDir.path
val targetList: List<String> by lazy {
targetsDir.listFiles.map{it.name}
}
}
fun KonanLibrary(klib: File, target: KonanTarget? = null) =
if (klib.isFile) ZippedKonanLibrary(klib, target)
else UnzippedKonanLibrary(klib, target)
val KonanLibrary.realFiles
get() = when (this) {
is ZippedKonanLibrary -> FileExtractor(this)
// Unpacked library just provides its own files.
is UnzippedKonanLibrary -> this
else -> error("Provide an extractor for your container.")
}
@@ -16,66 +16,28 @@
package org.jetbrains.kotlin.backend.konan.library.impl
import org.jetbrains.kotlin.backend.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.KonanLibraryReader
import org.jetbrains.kotlin.backend.konan.library.MetadataReader
import org.jetbrains.kotlin.backend.konan.serialization.deserializeModule
import org.jetbrains.kotlin.backend.konan.util.*
import org.jetbrains.kotlin.backend.konan.util.File
import org.jetbrains.kotlin.backend.konan.util.Properties
import org.jetbrains.kotlin.backend.konan.util.loadProperties
import org.jetbrains.kotlin.backend.konan.util.propertyList
import org.jetbrains.kotlin.config.LanguageVersionSettings
import org.jetbrains.kotlin.konan.target.KonanTarget
abstract class FileBasedLibraryReader(
val file: File, val currentAbiVersion: Int,
val reader: MetadataReader): KonanLibraryReader {
class LibraryReaderImpl(var libraryFile: File, val currentAbiVersion: Int, val target: KonanTarget? = null) : KonanLibraryReader {
override val libraryName: String
get() = file.path
// For the zipped libraries inPlace gives files from zip file system
// whereas realFiles extracts them to /tmp.
// For unzipped libraries inPlace and realFiles are the same
// providing files in the library directory.
private val inPlace = KonanLibrary(libraryFile, target)
private val realFiles = inPlace.realFiles
val moduleHeaderData: ByteArray by lazy {
reader.loadSerializedModule()
}
fun packageMetadata(fqName: String): ByteArray =
reader.loadSerializedPackageFragment(fqName)
override fun moduleDescriptor(specifics: LanguageVersionSettings)
= deserializeModule(specifics, {packageMetadata(it)},
moduleHeaderData)
}
class LibraryReaderImpl(override val libDir: File, currentAbiVersion: Int,
override val target: KonanTarget?) :
FileBasedLibraryReader(libDir, currentAbiVersion, MetadataReaderImpl(libDir)),
KonanLibrary {
public constructor(path: String, currentAbiVersion: Int, target: KonanTarget?) :
this(File(path), currentAbiVersion, target)
init {
unpackIfNeeded()
}
// TODO: Search path processing is also goes somewhere around here.
fun unpackIfNeeded() {
// TODO: Clarify the policy here.
if (libDir.exists) {
if (libDir.isDirectory) return
}
if (!klibFile.exists) {
error("Could not find neither $libDir nor $klibFile.")
}
if (klibFile.isFile) {
klibFile.unzipAs(libDir)
if (!libDir.exists) error("Could not unpack $klibFile as $libDir.")
} else {
error("Expected $klibFile to be a regular file.")
}
}
private val reader = MetadataReaderImpl(inPlace)
val manifestProperties: Properties by lazy {
manifestFile.loadProperties()
inPlace.manifestFile.loadProperties()
}
val abiVersion: String
@@ -86,10 +48,26 @@ class LibraryReaderImpl(override val libDir: File, currentAbiVersion: Int,
return manifestAbiVersion
}
val targetList = inPlace.targetsDir.listFiles.map{it.name}
override val libraryName
get() = inPlace.libraryName
override val bitcodePaths: List<String>
get() = (kotlinDir.listFiles + nativeDir.listFiles).map{it.absolutePath}
get() = (realFiles.kotlinDir.listFiles + realFiles.nativeDir.listFiles).map{it.absolutePath}
override val linkerOpts: List<String>
get() = manifestProperties.propertyList("linkerOpts", target!!.targetSuffix)
val moduleHeaderData: ByteArray by lazy {
reader.loadSerializedModule()
}
fun packageMetadata(fqName: String): ByteArray =
reader.loadSerializedPackageFragment(fqName)
override fun moduleDescriptor(specifics: LanguageVersionSettings)
= deserializeModule(specifics, {packageMetadata(it)}, moduleHeaderData)
}
@@ -36,6 +36,10 @@ class LibraryWriterImpl(override val libDir: File, currentAbiVersion: Int,
target:KonanTarget?, nopack: Boolean):
this(File(path), currentAbiVersion, target, nopack)
override val libraryName = libDir.path
val klibFile
get() = File("${libDir.path}.klib")
// TODO: Experiment with separate bitcode files.
// Per package or per class.
val mainBitcodeFile = File(kotlinDir, "program.kt.bc")
@@ -63,7 +67,7 @@ class LibraryWriterImpl(override val libDir: File, currentAbiVersion: Int,
}
override fun addLinkData(linkData: LinkData) {
MetadataWriterImpl(libDir).addLinkData(linkData)
MetadataWriterImpl(this).addLinkData(linkData)
}
override fun addNativeBitcode(library: String) {
@@ -18,9 +18,8 @@ package org.jetbrains.kotlin.backend.konan.library.impl
import org.jetbrains.kotlin.backend.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.MetadataReader
import org.jetbrains.kotlin.backend.konan.util.File
class MetadataReaderImpl(override val libDir: File) : MetadataReader, KonanLibrary {
class MetadataReaderImpl(library: KonanLibrary) : MetadataReader, KonanLibrary by library {
override fun loadSerializedModule(): ByteArray {
return moduleHeaderFile.readBytes()
@@ -20,7 +20,7 @@ import org.jetbrains.kotlin.backend.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.LinkData
import org.jetbrains.kotlin.backend.konan.util.File
internal class MetadataWriterImpl(override val libDir: File): KonanLibrary {
internal class MetadataWriterImpl(library: KonanLibrary): KonanLibrary by library {
fun addLinkData(linkData: LinkData) {
@@ -20,53 +20,106 @@ import java.io.BufferedReader
import java.io.InputStream
import java.io.InputStreamReader
import java.net.URI
import java.nio.file.FileSystems
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.StandardCopyOption
import java.nio.file.*
import java.nio.file.attribute.BasicFileAttributes
class File(val path: String) {
private constructor(file: java.io.File): this(file.path)
constructor(parent: String, child: String): this(java.io.File(parent, child))
constructor(parent: File, child: String): this(java.io.File(parent.path, child))
class File constructor(internal val javaPath: Path) {
constructor(parent: Path, child: String): this(parent.resolve(child))
constructor(parent: File, child: String): this(parent.javaPath.resolve(child))
constructor(path: String): this(Paths.get(path))
constructor(parent: String, child: String): this(Paths.get(parent, child))
private val javaFile = java.io.File(path)
val path: String
get() = javaPath.toString()
val absolutePath: String
get() = javaFile.absolutePath
get() = javaPath.toAbsolutePath().toString()
val absoluteFile: File
get() = File(absolutePath)
val name: String
get() = javaFile.name
get() = javaPath.fileName.toString()
val parent: String
get() = javaFile.parent
get() = javaPath.parent.toString()
val exists
get() = javaFile.exists()
get() = Files.exists(javaPath)
val isDirectory
get() = javaFile.isDirectory()
get() = Files.isDirectory(javaPath)
val isFile
get() = javaFile.isFile()
get() = Files.isRegularFile(javaPath)
val isAbsolute
get() = javaFile.isAbsolute()
val listFiles
get() = javaFile.listFiles()!!.toList()
get() = javaPath.isAbsolute()
val listFiles: List<File>
get() {
val stream = Files.newDirectoryStream(javaPath)
val result = mutableListOf<File>()
for (entry in stream) {
result.add(File(entry))
}
return result
}
fun mkdirs() = javaFile.mkdirs()
fun delete() = javaFile.delete()
fun deleteRecursively() = javaFile.deleteRecursively()
fun deleteOnExit() = javaFile.deleteOnExit()
fun readText() = javaFile.readText()
fun readBytes() = javaFile.readBytes()
fun writeText(text: String) = javaFile.writeText(text)
fun writeBytes(bytes: ByteArray) = javaFile.writeBytes(bytes)
fun forEachLine(action: (String) -> Unit) { javaFile.forEachLine { action(it) } }
fun copyTo(destination: File, vararg options: StandardCopyOption) {
Files.copy(javaPath, destination.javaPath, StandardCopyOption.REPLACE_EXISTING)
}
fun recursiveCopyTo(destination: File) {
val sourcePath = javaPath
val destPath = destination.javaPath
sourcePath.recursiveCopyTo(destPath)
}
fun mkdirs() = Files.createDirectories(javaPath)
fun delete() = Files.deleteIfExists(javaPath)
fun deleteRecursively() = postorder{Files.delete(it)}
fun deleteOnExitRecursively() = preorder{File(it).deleteOnExit()}
fun preorder(task: (Path) -> Unit) {
if (!this.exists) return
Files.walkFileTree(javaPath, object: SimpleFileVisitor<Path>() {
override fun visitFile(file: Path?, attrs: BasicFileAttributes?): FileVisitResult {
task(file!!)
return FileVisitResult.CONTINUE
}
override fun preVisitDirectory(dir: Path?, attrs: BasicFileAttributes?): FileVisitResult {
task(dir!!)
return FileVisitResult.CONTINUE
}
})
}
fun postorder(task: (Path) -> Unit) {
if (!this.exists) return
Files.walkFileTree(javaPath, object: SimpleFileVisitor<Path>() {
override fun visitFile(file: Path?, attrs: BasicFileAttributes?): FileVisitResult {
task(file!!)
return FileVisitResult.CONTINUE
}
override fun postVisitDirectory(dir: Path?, exc: java.io.IOException?): FileVisitResult {
task(dir!!)
return FileVisitResult.CONTINUE
}
})
}
fun deleteOnExit() {
// Works only on the default file system,
// but that's okay for now.
javaPath.toFile().deleteOnExit()
}
fun readBytes() = Files.readAllBytes(javaPath)
fun writeBytes(bytes: ByteArray) = Files.write(javaPath, bytes)
fun forEachLine(action: (String) -> Unit) { Files.lines(javaPath).forEach { action(it) } }
override fun toString() = path
// TODO: Consider removeing these after konanazing java.util.Properties.
fun bufferedReader() = javaFile.bufferedReader()
fun outputStream() = javaFile.outputStream()
fun bufferedReader() = Files.newBufferedReader(javaPath)
fun outputStream() = Files.newOutputStream(javaPath)
companion object {
val userDir
@@ -75,38 +128,44 @@ class File(val path: String) {
val userHome
get() = File(System.getProperty("user.home"))
fun createTempFile(name: String, suffix: String? = null, directory: File? = null): File {
val javaDirectory = directory ?.let { java.io.File(directory.path) }
return java.io.File.createTempFile(name, suffix, javaDirectory).path.File()
}
}
}
fun String.File() = File(this)
fun String.File(): File = File(this)
fun Path.File(): File = File(this)
fun createTempFile(name: String, suffix: String? = null)
= Files.createTempFile(name, suffix).File()
fun createTempDir(name: String): File
= Files.createTempDirectory(name).File()
private val File.zipUri: URI
get() = URI.create("jar:${this.toPath().toUri()}")
private val File.zipRootPath: Path
get() {
val zipUri = this.zipUri
val allowCreation = hashMapOf("create" to "true")
val zipfs = FileSystems.newFileSystem(zipUri, allowCreation, null)
return zipfs.getPath("/")
}
fun File.zipFileSystem(mutable: Boolean = false): FileSystem {
val zipUri = this.zipUri
val allowCreation = if (mutable) "true" else "false"
val attributes = hashMapOf("create" to if (mutable) "true" else "false")
return FileSystems.newFileSystem(zipUri, attributes, null)
}
fun File.mutableZipFileSystem() = this.zipFileSystem(mutable = true)
fun File.zipPath(path: String): Path
= this.zipFileSystem().getPath(path)
val File.asZipRoot: File
get() = File(this.zipPath("/"))
val File.asWritableZipRoot: File
get() = File(this.mutableZipFileSystem().getPath("/"))
private fun File.toPath() = Paths.get(this.path)
fun File.zipDirAs(zipFile: File) {
val zipPath = zipFile.zipRootPath
this.toPath().recursiveCopyTo(zipPath)
zipPath.fileSystem.close()
}
fun File.unzipAs(directory: File) {
val zipPath = this.zipRootPath
zipPath.recursiveCopyTo(directory.toPath())
zipPath.fileSystem.close()
fun File.zipDirAs(unixFile: File) {
val zipRoot = unixFile.asWritableZipRoot
this.recursiveCopyTo(zipRoot)
zipRoot.javaPath.fileSystem.close()
}
fun Path.recursiveCopyTo(destPath: Path) {
@@ -126,8 +185,5 @@ fun Path.recursiveCopyTo(destPath: Path) {
}
}
fun File.copyTo(destination: File, vararg options: StandardCopyOption) {
Files.copy(this.toPath(), destination.toPath(), *options)
}
fun bufferedReader(errorStream: InputStream?) = BufferedReader(InputStreamReader(errorStream))
fun bufferedReader(errorStream: InputStream?) = BufferedReader(InputStreamReader(errorStream))
@@ -19,10 +19,10 @@ package org.jetbrains.kotlin.cli.klib
import kotlin.system.exitProcess
import java.util.Properties
// TODO: Extract these as a shared jar?
import org.jetbrains.kotlin.backend.konan.library.impl.LibraryReaderImpl
import org.jetbrains.kotlin.backend.konan.library.impl.*
import org.jetbrains.kotlin.backend.konan.library.KonanLibrary
import org.jetbrains.kotlin.backend.konan.library.KonanLibrarySearchPathResolver
import org.jetbrains.kotlin.backend.konan.util.File
import org.jetbrains.kotlin.backend.konan.util.copyTo
import org.jetbrains.kotlin.konan.target.TargetManager
fun printUsage() {
@@ -83,53 +83,49 @@ class Library(val name: String, val requestedRepository: String?, val target: St
val repository = requestedRepository?.File() ?: defaultRepository
fun info() {
val library = libraryInRepoOrCurrentDir(repository, name)
val header = Properties()
val manifestFile = library.manifestFile
manifestFile.bufferedReader().use { reader ->
header.load(reader)
}
val headerAbiVersion = library.abiVersion
val moduleName = ModuleDeserializer(library.moduleHeaderData).moduleName
val reader = LibraryReaderImpl(library, currentAbiVersion)
val header = reader.manifestProperties
val headerAbiVersion = reader.abiVersion
val moduleName = ModuleDeserializer(reader.moduleHeaderData).moduleName
println("")
println("Resolved to: ${library.libDir.absolutePath}")
println("Resolved to: ${reader.libraryName.File().absolutePath}")
println("Module name: $moduleName")
println("ABI version: $headerAbiVersion")
val targets = library.targetsDir.listFiles.map{it.name}.joinToString(", ")
val targets = reader.targetList.joinToString(", ")
print("Available targets: $targets\n")
}
fun install() {
remove(true)
val klibFile = libraryInCurrentDir(name).klibFile
val library = ZippedKonanLibrary(libraryInCurrentDir(name))
val newLocation = File(repository, "klib")
newLocation.mkdirs()
val newFile = File(newLocation, klibFile.name)
klibFile.copyTo(newFile)
val newLibDir = File(newLocation, library.libraryName.File().name)
newLibDir.mkdirs()
library.unpackTo(newLibDir)
}
fun remove(blind: Boolean = false) {
if (!repository.exists) error("Repository does not exist: $repository")
val library = try {
val reader = try {
val library = libraryInRepo(repository, name)
if (blind) warn("Removing The previously installed $name from $repository.")
library
UnzippedKonanLibrary(library)
} catch (e: Throwable) {
if (!blind) println(e.message)
null
}
library?.libDir?.deleteRecursively()
library?.klibFile?.delete()
reader?.libDir?.deleteRecursively()
}
fun contents() {
val library = libraryInRepoOrCurrentDir(repository, name)
val reader = LibraryReaderImpl(libraryInRepoOrCurrentDir(repository, name), currentAbiVersion)
val printer = PrettyPrinter(
library.moduleHeaderData, {name -> library.packageMetadata(name)})
reader.moduleHeaderData, {name -> reader.packageMetadata(name)})
printer.packageFragmentNameList.forEach{
printer.printPackageFragment(it)
@@ -140,22 +136,19 @@ class Library(val name: String, val requestedRepository: String?, val target: St
// TODO: need to do something here.
val currentAbiVersion = 1
val File.konanLibrary
get() = LibraryReaderImpl(this, currentAbiVersion, null)
fun libraryInRepo(repository: File, name: String): LibraryReaderImpl {
fun libraryInRepo(repository: File, name: String): File {
val resolver = KonanLibrarySearchPathResolver(listOf(repository.absolutePath), null, null, skipCurrentDir = true)
return resolver.resolve(name).konanLibrary
return resolver.resolve(name)
}
fun libraryInCurrentDir(name: String): LibraryReaderImpl {
fun libraryInCurrentDir(name: String): File {
val resolver = KonanLibrarySearchPathResolver(emptyList(), null, null)
return resolver.resolve(name).konanLibrary
return resolver.resolve(name)
}
fun libraryInRepoOrCurrentDir(repository: File, name: String): LibraryReaderImpl {
fun libraryInRepoOrCurrentDir(repository: File, name: String): File {
val resolver = KonanLibrarySearchPathResolver(listOf(repository.absolutePath), null, null)
return resolver.resolve(name).konanLibrary
return resolver.resolve(name)
}