diff --git a/build.gradle.kts b/build.gradle.kts
index c91f68211ae..bf8bc0385d8 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -565,10 +565,13 @@ allprojects {
     }
 }
 
-gradle.buildFinished {
-    val taskGraph = gradle?.taskGraph
-    if (taskGraph != null) {
-        taskGraph.allTasks
+if ((gradle.startParameter as? org.gradle.api.internal.StartParameterInternal)?.isConfigurationCache != true) {
+    // TODO: remove it once Gradle is bumped to 6.8:
+    // See https://docs.gradle.org/6.8/release-notes.html#more-cache-hits-when-empty-directories-are-present
+    gradle.buildFinished {
+        val taskGraph = gradle?.taskGraph
+        if (taskGraph != null) {
+            taskGraph.allTasks
                 .filterIsInstance<SourceTask>()
                 .filter { it.didWork }
                 .forEach {
@@ -578,6 +581,7 @@ gradle.buildFinished {
                         }
                     }
                 }
+        }
     }
 }
 
diff --git a/gradle/report.gradle.kts b/gradle/report.gradle.kts
index 93241645f2e..03d7d6cc043 100644
--- a/gradle/report.gradle.kts
+++ b/gradle/report.gradle.kts
@@ -10,7 +10,9 @@ import org.gradle.api.logging.Logger
 import org.gradle.api.tasks.TaskState
 import java.util.HashMap
 
-BuildTimeReporter.configure(gradle)
+if ((gradle.startParameter as? org.gradle.api.internal.StartParameterInternal)?.isConfigurationCache != true) {
+    BuildTimeReporter.configure(gradle)
+}
 
 private class BuildTimeReporter(
     private val kotlinCompileClass: Class<*>,