From eabfd142f3bdb99aecf73d2641be6397aee51f13 Mon Sep 17 00:00:00 2001 From: Alexey Tsvetkov Date: Tue, 15 Sep 2015 17:25:28 +0300 Subject: [PATCH] Create Kotlin incremental caches only on write Original commit: fe390a04c512473c8e70c89f78cba603e2b91226 --- .../jps/incremental/IncrementalCacheImpl.kt | 64 +++++----- .../jps/incremental/storage/BasicMap.kt | 53 +++----- .../jps/incremental/storage/LazyStorage.kt | 113 ++++++++++++++++++ .../kotlin/jps/build/KotlinJpsBuildTest.kt | 11 ++ .../kotlinProject.iml | 13 ++ .../kotlinProject.ipr | 15 +++ .../module2/module2.iml | 14 +++ .../module2/src/B.java | 2 + .../src/A.java | 2 + .../src/utils.kt | 1 + 10 files changed, 217 insertions(+), 71 deletions(-) create mode 100644 jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/LazyStorage.kt create mode 100644 jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/kotlinProject.iml create mode 100644 jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/kotlinProject.ipr create mode 100644 jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/module2/module2.iml create mode 100644 jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/module2/src/B.java create mode 100644 jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/src/A.java create mode 100644 jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/src/utils.kt diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt index d9a9c37e082..4171936452b 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt @@ -92,7 +92,7 @@ class CacheFormatVersion(targetDataRoot: File) { } fun saveIfNeeded() { - if (!file.exists()) { + if (file.parentFile.exists() && !file.exists()) { file.writeText(actualCacheFormatVersion().toString()) } } @@ -271,9 +271,8 @@ public class IncrementalCacheImpl( .toList() val changesInfo = dirtyClasses.fold(ChangesInfo.NO_CHANGES) { info, className -> - val internalName = className.internalName - val newInfo = ChangesInfo(protoChanged = internalName in protoMap, - constantsChanged = internalName in constantsMap) + val newInfo = ChangesInfo(protoChanged = className in protoMap, + constantsChanged = className in constantsMap) newInfo.logIfSomethingChanged(className) info + newInfo } @@ -334,7 +333,7 @@ public class IncrementalCacheImpl( val data = ProtoMapValue(isPackage, bytes) if (oldData == null || !Arrays.equals(bytes, oldData.bytes) || isPackage != oldData.isPackageFacade) { - storage.put(key, data) + storage[key] = data } return ChangesInfo(protoChanged = oldData == null || @@ -342,12 +341,14 @@ public class IncrementalCacheImpl( difference(oldData, data) != DifferenceKind.NONE) } - public fun get(className: JvmClassName): ProtoMapValue? { - return storage[className.getInternalName()] - } + public fun contains(className: JvmClassName): Boolean = + className.internalName in storage + + public fun get(className: JvmClassName): ProtoMapValue? = + storage[className.internalName] public fun remove(className: JvmClassName) { - storage.remove(className.getInternalName()) + storage.remove(className.internalName) } override fun dumpValue(value: ProtoMapValue): String { @@ -372,6 +373,9 @@ public class IncrementalCacheImpl( return if (result.isEmpty()) null else result } + fun contains(className: JvmClassName): Boolean = + className.internalName in storage + public fun process(kotlinClass: LocalFileKotlinClass): ChangesInfo { return put(kotlinClass.className, getConstantsMap(kotlinClass.fileContents)) } @@ -383,7 +387,7 @@ public class IncrementalCacheImpl( if (oldMap == constantsMap) return ChangesInfo.NO_CHANGES if (constantsMap != null) { - storage.put(key, constantsMap) + storage[key] = constantsMap } else { storage.remove(key) @@ -514,7 +518,7 @@ public class IncrementalCacheImpl( } when { - newMap.isNotEmpty() -> storage.put(internalName, newMap) + newMap.isNotEmpty() -> storage[internalName] = newMap else -> storage.remove(internalName) } @@ -536,48 +540,46 @@ public class IncrementalCacheImpl( private inner class PackagePartMap(storageFile: File) : BasicStringMap(storageFile, BooleanDataDescriptor.INSTANCE) { public fun addPackagePart(className: JvmClassName) { - storage.put(className.getInternalName(), true) + storage[className.internalName] = true } public fun remove(className: JvmClassName) { - storage.remove(className.getInternalName()) + storage.remove(className.internalName) } - public fun isPackagePart(className: JvmClassName): Boolean { - return storage.containsMapping(className.getInternalName()) - } + public fun isPackagePart(className: JvmClassName): Boolean = + className.internalName in storage override fun dumpValue(value: Boolean) = "" } private inner class SourceToClassesMap(storageFile: File) : BasicStringMap>(storageFile, PathStringDescriptor.INSTANCE, StringListExternalizer) { public fun clearOutputsForSource(sourceFile: File) { - storage.remove(sourceFile.getAbsolutePath()) + storage.remove(sourceFile.absolutePath) } public fun add(sourceFile: File, className: JvmClassName) { - storage.appendData(sourceFile.getAbsolutePath(), { out -> IOUtil.writeUTF(out, className.getInternalName()) }) + storage.append(sourceFile.absolutePath, { out -> IOUtil.writeUTF(out, className.getInternalName()) }) } - public fun get(sourceFile: File): Collection { - return storage[sourceFile.getAbsolutePath()].orEmpty().map { JvmClassName.byInternalName(it) } - } + public fun get(sourceFile: File): Collection = + storage[sourceFile.absolutePath].orEmpty().map { JvmClassName.byInternalName(it) } override fun dumpValue(value: List) = value.toString() } private inner class ClassToSourcesMap(storageFile: File) : BasicStringMap>(storageFile, PathCollectionExternalizer) { public fun get(className: JvmClassName): Collection = - storage[className.internalName] ?: emptySet() + getFromStorage(className.internalName) ?: emptySet() public fun add(className: JvmClassName, sourceFile: File) { - storage.appendData(className.internalName) { out -> + appendDataToStorage(className.internalName) { out -> IOUtil.writeUTF(out, sourceFile.normalizedPath) } } public fun remove(className: JvmClassName) { - storage.remove(className.internalName) + removeFromStorage(className.internalName) } override fun dumpValue(value: Collection): String = @@ -586,27 +588,25 @@ public class IncrementalCacheImpl( private inner class DirtyOutputClassesMap(storageFile: File) : BasicStringMap(storageFile, BooleanDataDescriptor.INSTANCE) { public fun markDirty(className: String) { - storage.put(className, true) + storage[className] = true } public fun notDirty(className: String) { storage.remove(className) } - public fun getDirtyOutputClasses(): Collection { - return storage.getAllKeysWithExistingMapping() - } + public fun getDirtyOutputClasses(): Collection = + storage.keys override fun dumpValue(value: Boolean) = "" } private inner class DirtyInlineFunctionsMap(storageFile: File) : BasicStringMap>(storageFile, StringListExternalizer) { public fun getEntries(): Map> = - storage.allKeysWithExistingMapping - .toMap(JvmClassName::byInternalName) { storage[it] } + storage.keys.toMap(JvmClassName::byInternalName) { storage[it]!! } public fun put(className: JvmClassName, changedFunctions: List) { - storage.put(className.internalName, changedFunctions) + storage[className.internalName] = changedFunctions } override fun dumpValue(value: List) = @@ -625,7 +625,7 @@ public class IncrementalCacheImpl( private inner class InlineFunctionsFilesMap(storageFile: File) : BasicMap>(storageFile, PathFunctionPairKeyDescriptor, PathCollectionExternalizer) { public fun add(sourcePath: String, jvmSignature: String, targetPath: String) { val key = PathFunctionPair(sourcePath, jvmSignature) - storage.appendData(key) { out -> + storage.append(key) { out -> IOUtil.writeUTF(out, targetPath) } } diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/BasicMap.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/BasicMap.kt index 7ef01948a49..13d62fc9e42 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/BasicMap.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/BasicMap.kt @@ -19,60 +19,38 @@ package org.jetbrains.kotlin.jps.incremental.storage import com.intellij.util.io.DataExternalizer import com.intellij.util.io.EnumeratorStringDescriptor import com.intellij.util.io.KeyDescriptor -import com.intellij.util.io.PersistentHashMap import org.jetbrains.annotations.TestOnly import org.jetbrains.kotlin.utils.Printer import java.io.File -import java.io.IOException -public abstract class BasicMap, V>( - private val storageFile: File, - private val keyDescriptor: KeyDescriptor, - private val valueExternalizer: DataExternalizer +abstract class BasicMap, V>( + storageFile: File, + keyDescriptor: KeyDescriptor, + valueExternalizer: DataExternalizer ) { - protected var storage: PersistentHashMap = createMap() + protected val storage = LazyStorage(storageFile, keyDescriptor, valueExternalizer) - public fun contains(key: K): Boolean = storage.containsMapping(key) - - public fun clean() { - try { - storage.close() - } - catch (ignored: IOException) { - } - - PersistentHashMap.deleteFilesStartingWith(storage.getBaseFile()!!) - try { - storage = createMap() - } - catch (ignored: IOException) { - } + fun clean() { + storage.clean() } - public fun flush(memoryCachesOnly: Boolean) { - if (memoryCachesOnly) { - if (storage.isDirty()) { - storage.dropMemoryCaches() - } - } - else { - storage.force() - } + fun flush(memoryCachesOnly: Boolean) { + storage.flush(memoryCachesOnly) } - public fun close() { + fun close() { storage.close() } @TestOnly - public fun dump(): String { + fun dump(): String { return with(StringBuilder()) { with(Printer(this)) { - println(this@BasicMap.javaClass.getSimpleName()) + println(this@BasicMap.javaClass.simpleName) pushIndent() - for (key in storage.getAllKeysWithExistingMapping().sort()) { - println("${dumpKey(key)} -> ${dumpValue(storage[key])}") + for (key in storage.keys.sorted()) { + println("${dumpKey(key)} -> ${dumpValue(storage[key]!!)}") } popIndent() @@ -84,9 +62,6 @@ public abstract class BasicMap, V>( protected abstract fun dumpKey(key: K): String protected abstract fun dumpValue(value: V): String - - private fun createMap(): PersistentHashMap = - PersistentHashMap(storageFile, keyDescriptor, valueExternalizer) } public abstract class BasicStringMap( diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/LazyStorage.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/LazyStorage.kt new file mode 100644 index 00000000000..9a002b05951 --- /dev/null +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/LazyStorage.kt @@ -0,0 +1,113 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.jps.incremental.storage + +import com.intellij.util.io.DataExternalizer +import com.intellij.util.io.KeyDescriptor +import com.intellij.util.io.PersistentHashMap +import java.io.DataOutput +import java.io.File +import java.io.IOException + + +/** + * It's lazy in a sense that PersistentHashMap is created only on write + */ +internal class LazyStorage( + private val storageFile: File, + private val keyDescriptor: KeyDescriptor, + private val valueExternalizer: DataExternalizer +) { + @Volatile + private var storage: PersistentHashMap? = null + + @Synchronized + private fun getStorageIfExists(): PersistentHashMap? { + if (storage != null) return storage + + if (storageFile.exists()) { + storage = createMap() + return storage + } + + return null + } + + @Synchronized + private fun getStorageOrCreateNew(): PersistentHashMap { + if (storage == null) { + storage = createMap() + } + + return storage!! + } + + val keys: Collection + get() = getStorageIfExists()?.allKeysWithExistingMapping ?: listOf() + + fun contains(key: K): Boolean = + getStorageIfExists()?.containsMapping(key) ?: false + + fun get(key: K): V? = + getStorageIfExists()?.get(key) + + fun set(key: K, value: V) { + getStorageOrCreateNew().put(key, value) + } + + fun remove(key: K) { + getStorageIfExists()?.remove(key) + } + + fun append(key: K, append: (DataOutput)->Unit) { + getStorageOrCreateNew().appendData(key, append) + } + + @Synchronized + fun clean() { + try { + storage?.close() + } + catch (ignored: IOException) { + } + + PersistentHashMap.deleteFilesStartingWith(storageFile) + storage = null + } + + @Synchronized + fun flush(memoryCachesOnly: Boolean) { + val existingStorage = storage ?: return + + if (memoryCachesOnly) { + if (existingStorage.isDirty) { + existingStorage.dropMemoryCaches() + } + } + else { + existingStorage.force() + } + } + + @Synchronized + fun close() { + storage?.close() + } + + private fun createMap(): PersistentHashMap = + PersistentHashMap(storageFile, keyDescriptor, valueExternalizer) +} diff --git a/jps/jps-plugin/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt b/jps/jps-plugin/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt index 4e34f068d9f..e43df4eb49e 100644 --- a/jps/jps-plugin/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt +++ b/jps/jps-plugin/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt @@ -651,6 +651,17 @@ public class KotlinJpsBuildTest : AbstractKotlinJpsBuildTestCase() { assertFalse(File(storageRoot, "targets/java-production/kotlinProject/kotlin").exists()) } + public fun testDoNotCreateUselessKotlinIncrementalCachesForDependentTargets() { + initProject() + makeAll().assertSuccessful() + + checkWhen(touch("src/utils.kt"), null, packageClasses("kotlinProject", "src/utils.kt", "_DefaultPackage")) + + val storageRoot = BuildDataPathsImpl(myDataStorageRoot).dataStorageRoot + assertTrue(File(storageRoot, "targets/java-production/kotlinProject/kotlin").exists()) + assertFalse(File(storageRoot, "targets/java-production/module2/kotlin").exists()) + } + private fun buildCustom(canceledStatus: CanceledStatus, logger: TestProjectBuilderLogger,buildResult: BuildResult) { val scopeBuilder = CompileScopeTestBuilder.make().all() val descriptor = this.createProjectDescriptor(BuildLoggingManager(logger)) diff --git a/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/kotlinProject.iml b/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/kotlinProject.iml new file mode 100644 index 00000000000..3b1ba79ed61 --- /dev/null +++ b/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/kotlinProject.iml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/kotlinProject.ipr b/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/kotlinProject.ipr new file mode 100644 index 00000000000..809737b0b76 --- /dev/null +++ b/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/kotlinProject.ipr @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + diff --git a/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/module2/module2.iml b/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/module2/module2.iml new file mode 100644 index 00000000000..eace69878a2 --- /dev/null +++ b/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/module2/module2.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/module2/src/B.java b/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/module2/src/B.java new file mode 100644 index 00000000000..66dd24ce675 --- /dev/null +++ b/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/module2/src/B.java @@ -0,0 +1,2 @@ +public class B { +} diff --git a/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/src/A.java b/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/src/A.java new file mode 100644 index 00000000000..61ff2abcc95 --- /dev/null +++ b/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/src/A.java @@ -0,0 +1,2 @@ +public class A { +} \ No newline at end of file diff --git a/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/src/utils.kt b/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/src/utils.kt new file mode 100644 index 00000000000..67d276629bf --- /dev/null +++ b/jps/jps-plugin/testData/general/DoNotCreateUselessKotlinIncrementalCachesForDependentTargets/src/utils.kt @@ -0,0 +1 @@ +fun f(): String = "f()" \ No newline at end of file