From 954a0117216e49c11985af0eb40b21c4a5b41046 Mon Sep 17 00:00:00 2001
From: Evgeny Gerashchenko <Evgeny.Gerashchenko@jetbrains.com>
Date: Wed, 24 Sep 2014 19:35:19 +0400
Subject: [PATCH] Added incremental cache format versioning.

---
 .../Incremental_Compilation_Tests.xml         |  9 ++--
 .../jet/generators/tests/GenerateTests.kt     |  3 +-
 .../jetbrains/jet/jps/build/KotlinBuilder.kt  | 16 ++++--
 .../jps/incremental/IncrementalCacheImpl.kt   | 39 ++++++++++++++-
 .../jps/build/AbstractIncrementalJpsTest.kt   | 10 +++-
 .../IncrementalCacheVersionChangedTest.kt     | 50 +++++++++++++++++++
 .../build/IncrementalJpsTestGenerated.java    |  6 ---
 .../custom/cacheVersionChanged/a.kt           |  4 ++
 .../custom/cacheVersionChanged/b.kt           |  4 ++
 .../custom/cacheVersionChanged/build.log      |  9 ++++
 .../cacheVersionChangedAndFileModified/a.kt   |  4 ++
 .../cacheVersionChangedAndFileModified/b.kt   |  4 ++
 .../b.kt.new                                  |  4 ++
 .../build.log                                 | 11 ++++
 14 files changed, 156 insertions(+), 17 deletions(-)
 create mode 100644 jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalCacheVersionChangedTest.kt
 create mode 100644 jps-plugin/testData/incremental/custom/cacheVersionChanged/a.kt
 create mode 100644 jps-plugin/testData/incremental/custom/cacheVersionChanged/b.kt
 create mode 100644 jps-plugin/testData/incremental/custom/cacheVersionChanged/build.log
 create mode 100644 jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/a.kt
 create mode 100644 jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/b.kt
 create mode 100644 jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/b.kt.new
 create mode 100644 jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/build.log

diff --git a/.idea/runConfigurations/Incremental_Compilation_Tests.xml b/.idea/runConfigurations/Incremental_Compilation_Tests.xml
index a8154ad45be..cf1600c1110 100644
--- a/.idea/runConfigurations/Incremental_Compilation_Tests.xml
+++ b/.idea/runConfigurations/Incremental_Compilation_Tests.xml
@@ -5,9 +5,9 @@
     <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
     <option name="ALTERNATIVE_JRE_PATH" value="" />
     <option name="PACKAGE_NAME" value="org.jetbrains.jet.jps.build" />
-    <option name="MAIN_CLASS_NAME" value="org.jetbrains.jet.jps.build.IncrementalJpsTestGenerated" />
+    <option name="MAIN_CLASS_NAME" value="" />
     <option name="METHOD_NAME" value="" />
-    <option name="TEST_OBJECT" value="class" />
+    <option name="TEST_OBJECT" value="pattern" />
     <option name="VM_PARAMETERS" value="-ea -Xmx512m -XX:MaxPermSize=320m -Dkotlin.incremental.compilation=true" />
     <option name="PARAMETERS" value="" />
     <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
@@ -19,7 +19,10 @@
     <envs>
       <env name="kotlin.tests.actually.compile" value="false" />
     </envs>
-    <patterns />
+    <patterns>
+      <pattern testClass="org.jetbrains.jet.jps.build.IncrementalJpsTestGenerated" />
+      <pattern testClass="org.jetbrains.jet.jps.build.IncrementalCacheVersionChangedTest" />
+    </patterns>
     <RunnerSettings RunnerId="Debug">
       <option name="DEBUG_PORT" value="" />
       <option name="TRANSPORT" value="0" />
diff --git a/generators/src/org/jetbrains/jet/generators/tests/GenerateTests.kt b/generators/src/org/jetbrains/jet/generators/tests/GenerateTests.kt
index fca950e4974..761875dc146 100644
--- a/generators/src/org/jetbrains/jet/generators/tests/GenerateTests.kt
+++ b/generators/src/org/jetbrains/jet/generators/tests/GenerateTests.kt
@@ -634,7 +634,8 @@ fun main(args: Array<String>) {
 
     testGroup("jps-plugin/test", "jps-plugin/testData") {
         testClass(javaClass<AbstractIncrementalJpsTest>()) {
-            model("incremental", extension = null, excludeParentDirs = true)
+            model("incremental/circularDependency", extension = null, excludeParentDirs = true)
+            model("incremental/pureKotlin", extension = null, excludeParentDirs = true)
         }
     }
 
diff --git a/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.kt
index 3efd7de51ec..86ca6b770d8 100644
--- a/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.kt
+++ b/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.kt
@@ -51,6 +51,7 @@ import org.jetbrains.jet.compiler.runner.KotlinCompilerRunner.runK2JvmCompiler
 import org.jetbrains.jet.utils.keysToMap
 import org.jetbrains.jps.incremental.ModuleLevelBuilder.ExitCode.*
 import com.intellij.openapi.diagnostic.Logger
+import org.jetbrains.jet.lang.resolve.java.JvmAbi
 
 public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
     class object {
@@ -83,6 +84,16 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
 
         val representativeTarget = chunk.representativeTarget()
 
+        val outputDir = representativeTarget.getOutputDir()
+
+        val dataManager = context.getProjectDescriptor().dataManager
+        val incrementalCaches = chunk.getTargets().keysToMap { dataManager.getStorage(it, IncrementalCacheStorageProvider) }
+
+        if (incrementalCaches.values().any { it -> it.isCacheVersionIncompatible() }) {
+            incrementalCaches.values().forEach { it.clean() }
+            return CHUNK_REBUILD_REQUIRED
+        }
+
         // For non-incremental build: take all sources
         if (!dirtyFilesHolder.hasDirtyFiles() && !dirtyFilesHolder.hasRemovedFiles()) {
             return NOTHING_DONE
@@ -94,11 +105,6 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
 
         messageCollector.report(INFO, "Kotlin JPS plugin version " + KotlinVersion.VERSION, NO_LOCATION)
 
-        val outputDir = representativeTarget.getOutputDir()
-
-        val dataManager = context.getProjectDescriptor().dataManager
-        val incrementalCaches = chunk.getTargets().keysToMap { dataManager.getStorage(it, IncrementalCacheStorageProvider) }
-
         val compilerServices = Services.Builder()
                 .register(javaClass<IncrementalCacheProvider>(), IncrementalCacheProviderImpl(incrementalCaches))
                 .build()
diff --git a/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementalCacheImpl.kt b/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementalCacheImpl.kt
index 580aaf752e2..cab43bc9fb0 100644
--- a/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementalCacheImpl.kt
+++ b/jps-plugin/src/org/jetbrains/jet/jps/incremental/IncrementalCacheImpl.kt
@@ -40,15 +40,25 @@ import java.security.MessageDigest
 import org.jetbrains.jps.incremental.storage.StorageOwner
 import org.jetbrains.jps.builders.storage.StorageProvider
 import java.io.IOException
+import java.util.Scanner
+import org.jetbrains.jet.lang.resolve.java.JvmAbi
 
 val INLINE_ANNOTATION_DESC = "Lkotlin/inline;"
 
 public class IncrementalCacheImpl(val baseDir: File): StorageOwner, IncrementalCache {
     class object {
+        val DIRECTORY_NAME = "kotlin"
+
         val PROTO_MAP = "proto.tab"
         val CONSTANTS_MAP = "constants.tab"
         val INLINE_FUNCTIONS = "inline-functions.tab"
         val PACKAGE_PARTS = "package-parts.tab"
+
+        // Change this when incremental cache format changes
+        private val INCREMENTAL_CACHE_OWN_VERSION = 1
+        public val CACHE_FORMAT_VERSION: Int = INCREMENTAL_CACHE_OWN_VERSION * 1000000 + JvmAbi.VERSION
+
+        val FORMAT_VERSION_TXT = "format-version.txt"
     }
 
     private val protoMap = ProtoMap()
@@ -58,8 +68,34 @@ public class IncrementalCacheImpl(val baseDir: File): StorageOwner, IncrementalC
 
     private val maps = listOf(protoMap, constantsMap, inlineFunctionsMap, packagePartMap)
 
+    private fun getFormatVersionFile(): File {
+        return File(baseDir, FORMAT_VERSION_TXT)
+    }
+
+    public fun isCacheVersionIncompatible(): Boolean {
+        val version = loadCacheFormatVersion()
+        return version != -1 && version != CACHE_FORMAT_VERSION
+    }
+
+    private fun loadCacheFormatVersion(): Int {
+        val versionFile = getFormatVersionFile()
+        if (!versionFile.exists()) return -1
+
+        return versionFile.readText().toInt()
+    }
+
+    private fun saveCacheFormatVersionIfNeeded() {
+        val versionFile = getFormatVersionFile()
+        if (!versionFile.exists()) {
+            versionFile.writeText(CACHE_FORMAT_VERSION.toString())
+        }
+    }
+
     public fun saveFileToCache(sourceFiles: Collection<File>, classFile: File): RecompilationDecision {
         if (classFile.extension.toLowerCase() != "class") return DO_NOTHING
+        
+        saveCacheFormatVersionIfNeeded()
+
         val kotlinClass = LocalFileKotlinClass.create(classFile)
         if (kotlinClass == null) return DO_NOTHING
 
@@ -121,6 +157,7 @@ public class IncrementalCacheImpl(val baseDir: File): StorageOwner, IncrementalC
 
     public override fun clean() {
         maps.forEach { it.clean() }
+        getFormatVersionFile().delete()
     }
 
     public override fun close() {
@@ -457,7 +494,7 @@ public class IncrementalCacheImpl(val baseDir: File): StorageOwner, IncrementalC
 
 public object IncrementalCacheStorageProvider : StorageProvider<IncrementalCacheImpl>() {
     override fun createStorage(targetDataDir: File?): IncrementalCacheImpl {
-        return IncrementalCacheImpl(File(targetDataDir, "kotlin"))
+        return IncrementalCacheImpl(File(targetDataDir, IncrementalCacheImpl.DIRECTORY_NAME))
     }
 }
 
diff --git a/jps-plugin/test/org/jetbrains/jet/jps/build/AbstractIncrementalJpsTest.kt b/jps-plugin/test/org/jetbrains/jet/jps/build/AbstractIncrementalJpsTest.kt
index 107886f8ea1..0a77dd7011f 100644
--- a/jps-plugin/test/org/jetbrains/jet/jps/build/AbstractIncrementalJpsTest.kt
+++ b/jps-plugin/test/org/jetbrains/jet/jps/build/AbstractIncrementalJpsTest.kt
@@ -50,6 +50,9 @@ public abstract class AbstractIncrementalJpsTest : JpsBuildTestCase() {
         super.tearDown()
     }
 
+    protected open val customTest: Boolean
+        get() = false
+
     fun buildGetLog(scope: CompileScopeTestBuilder = CompileScopeTestBuilder.make().all()): String {
         val logger = MyLogger(FileUtil.toSystemIndependentName(workDir.getAbsolutePath()))
         val descriptor = createProjectDescriptor(BuildLoggingManager(logger))
@@ -119,7 +122,12 @@ public abstract class AbstractIncrementalJpsTest : JpsBuildTestCase() {
             fail("Bad test data format: files ending with both unnumbered and numbered \".new\"/\".delete\" were found")
         }
         if (!haveFilesWithoutNumbers && !haveFilesWithNumbers) {
-            fail("Bad test data format: no files ending with \".new\" or \".delete\" found")
+            if (customTest) {
+                return listOf(listOf())
+            }
+            else {
+                fail("Bad test data format: no files ending with \".new\" or \".delete\" found")
+            }
         }
 
         if (haveFilesWithoutNumbers) {
diff --git a/jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalCacheVersionChangedTest.kt b/jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalCacheVersionChangedTest.kt
new file mode 100644
index 00000000000..8ad04c98423
--- /dev/null
+++ b/jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalCacheVersionChangedTest.kt
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2010-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.jet.jps.build
+
+import com.intellij.testFramework.TestDataPath
+import org.jetbrains.jet.JetTestUtils
+import com.intellij.openapi.util.io.FileUtil
+import org.jetbrains.jps.builders.impl.BuildDataPathsImpl
+import java.io.File
+import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType
+import org.jetbrains.jet.jps.incremental
+import com.sun.tools.javac.resources.version
+import org.jetbrains.jet.jps.incremental.IncrementalCacheImpl
+import kotlin.test.assertTrue
+
+public class IncrementalCacheVersionChangedTest : AbstractIncrementalJpsTest() {
+    fun testCacheVersionChanged() {
+        doTest("jps-plugin/testData/incremental/custom/cacheVersionChanged/")
+    }
+
+    fun testCacheVersionChangedAndFileModified() {
+        doTest("jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/")
+    }
+
+    override val customTest: Boolean
+        get() = true
+
+    override fun performAdditionalModifications() {
+        val storageForTargetType = BuildDataPathsImpl(myDataStorageRoot).getTargetTypeDataRoot(JavaModuleBuildTargetType.PRODUCTION)
+        val relativePath = "module/${File.separator}${IncrementalCacheImpl.DIRECTORY_NAME}/${IncrementalCacheImpl.FORMAT_VERSION_TXT}"
+        val cacheVersionFile = File(storageForTargetType, relativePath)
+
+        assertTrue(cacheVersionFile.exists())
+        cacheVersionFile.writeText("777")
+    }
+}
\ No newline at end of file
diff --git a/jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalJpsTestGenerated.java b/jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalJpsTestGenerated.java
index 2fae1f59adf..5a7c14ceebe 100644
--- a/jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalJpsTestGenerated.java
+++ b/jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalJpsTestGenerated.java
@@ -30,15 +30,9 @@ import java.util.regex.Pattern;
 
 /** This class is generated by {@link org.jetbrains.jet.generators.tests.TestsPackage}. DO NOT MODIFY MANUALLY */
 @SuppressWarnings("all")
-@TestMetadata("jps-plugin/testData/incremental")
-@TestDataPath("$PROJECT_ROOT")
 @InnerTestClasses({IncrementalJpsTestGenerated.CircularDependency.class, IncrementalJpsTestGenerated.PureKotlin.class})
 @RunWith(org.jetbrains.jet.JUnit3RunnerWithInners.class)
 public class IncrementalJpsTestGenerated extends AbstractIncrementalJpsTest {
-    public void testAllFilesPresentInIncremental() throws Exception {
-        JetTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental"), Pattern.compile("^([^\\.]+)$"), true);
-    }
-    
     @TestMetadata("jps-plugin/testData/incremental/circularDependency")
     @TestDataPath("$PROJECT_ROOT")
     @InnerTestClasses({})
diff --git a/jps-plugin/testData/incremental/custom/cacheVersionChanged/a.kt b/jps-plugin/testData/incremental/custom/cacheVersionChanged/a.kt
new file mode 100644
index 00000000000..91fe7016ac5
--- /dev/null
+++ b/jps-plugin/testData/incremental/custom/cacheVersionChanged/a.kt
@@ -0,0 +1,4 @@
+package test
+
+fun f() {
+}
\ No newline at end of file
diff --git a/jps-plugin/testData/incremental/custom/cacheVersionChanged/b.kt b/jps-plugin/testData/incremental/custom/cacheVersionChanged/b.kt
new file mode 100644
index 00000000000..c8d3d771251
--- /dev/null
+++ b/jps-plugin/testData/incremental/custom/cacheVersionChanged/b.kt
@@ -0,0 +1,4 @@
+package test
+
+fun g() {
+}
\ No newline at end of file
diff --git a/jps-plugin/testData/incremental/custom/cacheVersionChanged/build.log b/jps-plugin/testData/incremental/custom/cacheVersionChanged/build.log
new file mode 100644
index 00000000000..5811eb1266c
--- /dev/null
+++ b/jps-plugin/testData/incremental/custom/cacheVersionChanged/build.log
@@ -0,0 +1,9 @@
+Cleaning output files:
+out/production/module/test/TestPackage$a$*.class
+out/production/module/test/TestPackage$b$*.class
+out/production/module/test/TestPackage.class
+End of files
+Compiling files:
+src/a.kt
+src/b.kt
+End of files
\ No newline at end of file
diff --git a/jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/a.kt b/jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/a.kt
new file mode 100644
index 00000000000..91fe7016ac5
--- /dev/null
+++ b/jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/a.kt
@@ -0,0 +1,4 @@
+package test
+
+fun f() {
+}
\ No newline at end of file
diff --git a/jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/b.kt b/jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/b.kt
new file mode 100644
index 00000000000..c8d3d771251
--- /dev/null
+++ b/jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/b.kt
@@ -0,0 +1,4 @@
+package test
+
+fun g() {
+}
\ No newline at end of file
diff --git a/jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/b.kt.new b/jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/b.kt.new
new file mode 100644
index 00000000000..c8d3d771251
--- /dev/null
+++ b/jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/b.kt.new
@@ -0,0 +1,4 @@
+package test
+
+fun g() {
+}
\ No newline at end of file
diff --git a/jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/build.log b/jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/build.log
new file mode 100644
index 00000000000..073f1409ae5
--- /dev/null
+++ b/jps-plugin/testData/incremental/custom/cacheVersionChangedAndFileModified/build.log
@@ -0,0 +1,11 @@
+Cleaning output files:
+out/production/module/test/TestPackage$b$*.class
+out/production/module/test/TestPackage.class
+End of files
+Cleaning output files:
+out/production/module/test/TestPackage$a$*.class
+End of files
+Compiling files:
+src/a.kt
+src/b.kt
+End of files
\ No newline at end of file