Create Kotlin incremental caches only on write

Original commit: fe390a04c5
2015-09-15 17:25:28 +03:00
parent e810c6f007
commit eabfd142f3
10 changed files with 217 additions and 71 deletions
@@ -92,7 +92,7 @@ class CacheFormatVersion(targetDataRoot: File) {
    }

    fun saveIfNeeded() {
-        if (!file.exists()) {
+        if (file.parentFile.exists() && !file.exists()) {
            file.writeText(actualCacheFormatVersion().toString())
        }
    }
@@ -271,9 +271,8 @@ public class IncrementalCacheImpl(
                                .toList()

        val changesInfo = dirtyClasses.fold(ChangesInfo.NO_CHANGES) { info, className ->
-            val internalName = className.internalName
-            val newInfo = ChangesInfo(protoChanged = internalName in protoMap,
-                                      constantsChanged = internalName in constantsMap)
+            val newInfo = ChangesInfo(protoChanged = className in protoMap,
+                                      constantsChanged = className in constantsMap)
            newInfo.logIfSomethingChanged(className)
            info + newInfo
        }
@@ -334,7 +333,7 @@ public class IncrementalCacheImpl(
            val data = ProtoMapValue(isPackage, bytes)

            if (oldData == null || !Arrays.equals(bytes, oldData.bytes) || isPackage != oldData.isPackageFacade) {
-                storage.put(key, data)
+                storage[key] = data
            }

            return ChangesInfo(protoChanged = oldData == null ||
@@ -342,12 +341,14 @@ public class IncrementalCacheImpl(
                                              difference(oldData, data) != DifferenceKind.NONE)
        }

-        public fun get(className: JvmClassName): ProtoMapValue? {
-            return storage[className.getInternalName()]
-        }
+        public fun contains(className: JvmClassName): Boolean =
+                className.internalName in storage
+
+        public fun get(className: JvmClassName): ProtoMapValue? =
+                storage[className.internalName]

        public fun remove(className: JvmClassName) {
-            storage.remove(className.getInternalName())
+            storage.remove(className.internalName)
        }

        override fun dumpValue(value: ProtoMapValue): String {
@@ -372,6 +373,9 @@ public class IncrementalCacheImpl(
            return if (result.isEmpty()) null else result
        }

+        fun contains(className: JvmClassName): Boolean =
+                className.internalName in storage
+
        public fun process(kotlinClass: LocalFileKotlinClass): ChangesInfo {
            return put(kotlinClass.className, getConstantsMap(kotlinClass.fileContents))
        }
@@ -383,7 +387,7 @@ public class IncrementalCacheImpl(
            if (oldMap == constantsMap) return ChangesInfo.NO_CHANGES

            if (constantsMap != null) {
-                storage.put(key, constantsMap)
+                storage[key] = constantsMap
            }
            else {
                storage.remove(key)
@@ -514,7 +518,7 @@ public class IncrementalCacheImpl(
            }

            when {
-                newMap.isNotEmpty() -> storage.put(internalName, newMap)
+                newMap.isNotEmpty() -> storage[internalName] = newMap
                else -> storage.remove(internalName)
            }

@@ -536,48 +540,46 @@ public class IncrementalCacheImpl(

    private inner class PackagePartMap(storageFile: File) : BasicStringMap<Boolean>(storageFile, BooleanDataDescriptor.INSTANCE) {
        public fun addPackagePart(className: JvmClassName) {
-            storage.put(className.getInternalName(), true)
+            storage[className.internalName] = true
        }

        public fun remove(className: JvmClassName) {
-            storage.remove(className.getInternalName())
+            storage.remove(className.internalName)
        }

-        public fun isPackagePart(className: JvmClassName): Boolean {
-            return storage.containsMapping(className.getInternalName())
-        }
+        public fun isPackagePart(className: JvmClassName): Boolean =
+                className.internalName in storage

        override fun dumpValue(value: Boolean) = ""
    }

    private inner class SourceToClassesMap(storageFile: File) : BasicStringMap<List<String>>(storageFile, PathStringDescriptor.INSTANCE, StringListExternalizer) {
        public fun clearOutputsForSource(sourceFile: File) {
-            storage.remove(sourceFile.getAbsolutePath())
+            storage.remove(sourceFile.absolutePath)
        }

        public fun add(sourceFile: File, className: JvmClassName) {
-            storage.appendData(sourceFile.getAbsolutePath(), { out -> IOUtil.writeUTF(out, className.getInternalName()) })
+            storage.append(sourceFile.absolutePath, { out -> IOUtil.writeUTF(out, className.getInternalName()) })
        }

-        public fun get(sourceFile: File): Collection<JvmClassName> {
-            return storage[sourceFile.getAbsolutePath()].orEmpty().map { JvmClassName.byInternalName(it) }
-        }
+        public fun get(sourceFile: File): Collection<JvmClassName> =
+                storage[sourceFile.absolutePath].orEmpty().map { JvmClassName.byInternalName(it) }

        override fun dumpValue(value: List<String>) = value.toString()
    }

    private inner class ClassToSourcesMap(storageFile: File) : BasicStringMap<Collection<String>>(storageFile, PathCollectionExternalizer) {
        public fun get(className: JvmClassName): Collection<String> =
-                storage[className.internalName] ?: emptySet()
+                getFromStorage(className.internalName) ?: emptySet()

        public fun add(className: JvmClassName, sourceFile: File) {
-            storage.appendData(className.internalName) { out ->
+            appendDataToStorage(className.internalName) { out ->
                IOUtil.writeUTF(out, sourceFile.normalizedPath)
            }
        }

        public fun remove(className: JvmClassName) {
-            storage.remove(className.internalName)
+            removeFromStorage(className.internalName)
        }

        override fun dumpValue(value: Collection<String>): String =
@@ -586,27 +588,25 @@ public class IncrementalCacheImpl(

    private inner class DirtyOutputClassesMap(storageFile: File) : BasicStringMap<Boolean>(storageFile, BooleanDataDescriptor.INSTANCE) {
        public fun markDirty(className: String) {
-            storage.put(className, true)
+            storage[className] = true
        }

        public fun notDirty(className: String) {
            storage.remove(className)
        }

-        public fun getDirtyOutputClasses(): Collection<String> {
-            return storage.getAllKeysWithExistingMapping()
-        }
+        public fun getDirtyOutputClasses(): Collection<String> =
+                storage.keys

        override fun dumpValue(value: Boolean) = ""
    }

    private inner class DirtyInlineFunctionsMap(storageFile: File) : BasicStringMap<List<String>>(storageFile, StringListExternalizer) {
        public fun getEntries(): Map<JvmClassName, List<String>> =
-            storage.allKeysWithExistingMapping
-                   .toMap(JvmClassName::byInternalName) { storage[it] }
+            storage.keys.toMap(JvmClassName::byInternalName) { storage[it]!! }

        public fun put(className: JvmClassName, changedFunctions: List<String>) {
-            storage.put(className.internalName, changedFunctions)
+            storage[className.internalName] = changedFunctions
        }

        override fun dumpValue(value: List<String>) =
@@ -625,7 +625,7 @@ public class IncrementalCacheImpl(
    private inner class InlineFunctionsFilesMap(storageFile: File) : BasicMap<PathFunctionPair, Collection<String>>(storageFile, PathFunctionPairKeyDescriptor, PathCollectionExternalizer) {
        public fun add(sourcePath: String, jvmSignature: String, targetPath: String) {
            val key = PathFunctionPair(sourcePath, jvmSignature)
-            storage.appendData(key) { out ->
+            storage.append(key) { out ->
                IOUtil.writeUTF(out, targetPath)
            }
        }
@@ -19,60 +19,38 @@ package org.jetbrains.kotlin.jps.incremental.storage
 import com.intellij.util.io.DataExternalizer
 import com.intellij.util.io.EnumeratorStringDescriptor
 import com.intellij.util.io.KeyDescriptor
-import com.intellij.util.io.PersistentHashMap
 import org.jetbrains.annotations.TestOnly
 import org.jetbrains.kotlin.utils.Printer
 import java.io.File
-import java.io.IOException

-public abstract class BasicMap<K : Comparable<K>, V>(
-        private val storageFile: File,
-        private val keyDescriptor: KeyDescriptor<K>,
-        private val valueExternalizer: DataExternalizer<V>
+abstract class BasicMap<K : Comparable<K>, V>(
+        storageFile: File,
+        keyDescriptor: KeyDescriptor<K>,
+        valueExternalizer: DataExternalizer<V>
 ) {
-    protected var storage: PersistentHashMap<K, V> = createMap()
+    protected val storage = LazyStorage(storageFile, keyDescriptor, valueExternalizer)

-    public fun contains(key: K): Boolean = storage.containsMapping(key)
-
-    public fun clean() {
-        try {
-            storage.close()
-        }
-        catch (ignored: IOException) {
-        }
-
-        PersistentHashMap.deleteFilesStartingWith(storage.getBaseFile()!!)
-        try {
-            storage = createMap()
-        }
-        catch (ignored: IOException) {
-        }
+    fun clean() {
+        storage.clean()
    }

-    public fun flush(memoryCachesOnly: Boolean) {
-        if (memoryCachesOnly) {
-            if (storage.isDirty()) {
-                storage.dropMemoryCaches()
-            }
-        }
-        else {
-            storage.force()
-        }
+    fun flush(memoryCachesOnly: Boolean) {
+        storage.flush(memoryCachesOnly)
    }

-    public fun close() {
+    fun close() {
        storage.close()
    }

    @TestOnly
-    public fun dump(): String {
+    fun dump(): String {
        return with(StringBuilder()) {
            with(Printer(this)) {
-                println(this@BasicMap.javaClass.getSimpleName())
+                println(this@BasicMap.javaClass.simpleName)
                pushIndent()

-                for (key in storage.getAllKeysWithExistingMapping().sort()) {
-                    println("${dumpKey(key)} -> ${dumpValue(storage[key])}")
+                for (key in storage.keys.sorted()) {
+                    println("${dumpKey(key)} -> ${dumpValue(storage[key]!!)}")
                }

                popIndent()
@@ -84,9 +62,6 @@ public abstract class BasicMap<K : Comparable<K>, V>(

    protected abstract fun dumpKey(key: K): String
    protected abstract fun dumpValue(value: V): String
-
-    private fun createMap(): PersistentHashMap<K, V> =
-            PersistentHashMap(storageFile, keyDescriptor, valueExternalizer)
 }

 public abstract class BasicStringMap<V>(
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.jps.incremental.storage
+
+import com.intellij.util.io.DataExternalizer
+import com.intellij.util.io.KeyDescriptor
+import com.intellij.util.io.PersistentHashMap
+import java.io.DataOutput
+import java.io.File
+import java.io.IOException
+
+
+/**
+ * It's lazy in a sense that PersistentHashMap is created only on write
+ */
+internal class LazyStorage<K, V>(
+        private val storageFile: File,
+        private val keyDescriptor: KeyDescriptor<K>,
+        private val valueExternalizer: DataExternalizer<V>
+) {
+    @Volatile
+    private var storage: PersistentHashMap<K, V>? = null
+
+    @Synchronized
+    private fun getStorageIfExists(): PersistentHashMap<K, V>? {
+        if (storage != null) return storage
+
+        if (storageFile.exists()) {
+            storage = createMap()
+            return storage
+        }
+
+        return null
+    }
+
+    @Synchronized
+    private fun getStorageOrCreateNew(): PersistentHashMap<K, V> {
+        if (storage == null) {
+            storage = createMap()
+        }
+
+        return storage!!
+    }
+
+    val keys: Collection<K>
+        get() = getStorageIfExists()?.allKeysWithExistingMapping ?: listOf()
+
+    fun contains(key: K): Boolean =
+            getStorageIfExists()?.containsMapping(key) ?: false
+
+    fun get(key: K): V? =
+            getStorageIfExists()?.get(key)
+
+    fun set(key: K, value: V) {
+        getStorageOrCreateNew().put(key, value)
+    }
+
+    fun remove(key: K) {
+        getStorageIfExists()?.remove(key)
+    }
+
+    fun append(key: K, append: (DataOutput)->Unit) {
+        getStorageOrCreateNew().appendData(key, append)
+    }
+
+    @Synchronized
+    fun clean() {
+        try {
+            storage?.close()
+        }
+        catch (ignored: IOException) {
+        }
+
+        PersistentHashMap.deleteFilesStartingWith(storageFile)
+        storage = null
+    }
+
+    @Synchronized
+    fun flush(memoryCachesOnly: Boolean) {
+        val existingStorage = storage ?: return
+
+        if (memoryCachesOnly) {
+            if (existingStorage.isDirty) {
+                existingStorage.dropMemoryCaches()
+            }
+        }
+        else {
+            existingStorage.force()
+        }
+    }
+
+    @Synchronized
+    fun close() {
+        storage?.close()
+    }
+
+    private fun createMap(): PersistentHashMap<K, V> =
+            PersistentHashMap(storageFile, keyDescriptor, valueExternalizer)
+}
@@ -651,6 +651,17 @@ public class KotlinJpsBuildTest : AbstractKotlinJpsBuildTestCase() {
        assertFalse(File(storageRoot, "targets/java-production/kotlinProject/kotlin").exists())
    }

+    public fun testDoNotCreateUselessKotlinIncrementalCachesForDependentTargets() {
+        initProject()
+        makeAll().assertSuccessful()
+
+        checkWhen(touch("src/utils.kt"), null, packageClasses("kotlinProject", "src/utils.kt", "_DefaultPackage"))
+
+        val storageRoot = BuildDataPathsImpl(myDataStorageRoot).dataStorageRoot
+        assertTrue(File(storageRoot, "targets/java-production/kotlinProject/kotlin").exists())
+        assertFalse(File(storageRoot, "targets/java-production/module2/kotlin").exists())
+    }
+
    private fun buildCustom(canceledStatus: CanceledStatus, logger: TestProjectBuilderLogger,buildResult: BuildResult) {
        val scopeBuilder = CompileScopeTestBuilder.make().all()
        val descriptor = this.createProjectDescriptor(BuildLoggingManager(logger))
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+    </content>
+    <orderEntry type="jdk" jdkName="IDEA_JDK" jdkType="JavaSDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="KotlinRuntime" level="project" />
+  </component>
+</module>
+
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CompilerConfiguration">
+    <option name="DEFAULT_COMPILER" value="Javac" />
+  </component>
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/kotlinProject.iml" filepath="$PROJECT_DIR$/kotlinProject.iml" />
+      <module fileurl="file://$PROJECT_DIR$/module2/module2.iml" filepath="$PROJECT_DIR$/module2/module2.iml" />
+    </modules>
+  </component>
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_6" assert-keyword="true" jdk-15="true" project-jdk-name="IDEA_JDK" project-jdk-type="JavaSDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+    </content>
+    <orderEntry type="jdk" jdkName="IDEA_JDK" jdkType="JavaSDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="module" module-name="kotlinProject" />
+    <orderEntry type="library" name="KotlinRuntime" level="project" />
+  </component>
+</module>
+
@@ -0,0 +1,2 @@
+public class B {
+}
@@ -0,0 +1,2 @@
+public class A {
+}
@@ -0,0 +1 @@
+fun f(): String = "f()"