Implement source sections compiler plugin

allows to compile only parts of the source files, denoted by top-level
"sections" (function with lambda param calls), but preserving original
file line/column numbers for easier diagnostics. Allow e.g. to compile
gradle "buildscript" section without preprocessing original file in
advance. See tests for examples.

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/CompileEnvironmentUtil.java

@@ -41,6 +41,7 @@ import org.jetbrains.kotlin.cli.common.modules.ModuleScriptData;
 import org.jetbrains.kotlin.cli.common.modules.ModuleXmlParser;
 import org.jetbrains.kotlin.config.CompilerConfiguration;
 import org.jetbrains.kotlin.config.JVMConfigurationKeys;
+import org.jetbrains.kotlin.extensions.PreprocessedFileCreator;
 import org.jetbrains.kotlin.idea.KotlinFileType;
 import org.jetbrains.kotlin.name.FqName;
 import org.jetbrains.kotlin.psi.KtFile;
@@ -161,6 +162,8 @@ public class CompileEnvironmentUtil {
         final Set<VirtualFile> processedFiles = Sets.newHashSet();
         final List<KtFile> result = Lists.newArrayList();
 
+        final PreprocessedFileCreator virtualFileCreator = new PreprocessedFileCreator(project);
+
         for (String sourceRootPath : sourceRoots) {
             if (sourceRootPath == null) {
                 continue;
@@ -190,7 +193,8 @@ public class CompileEnvironmentUtil {
                 @Override
                 public Unit invoke(File file) {
                     if (file.isFile()) {
-                        VirtualFile virtualFile = localFileSystem.findFileByPath(file.getAbsolutePath());
+                        VirtualFile originalVirtualFile = localFileSystem.findFileByPath(file.getAbsolutePath());
+                        VirtualFile virtualFile = originalVirtualFile != null ? virtualFileCreator.create(originalVirtualFile) : null;
                         if (virtualFile != null && !processedFiles.contains(virtualFile)) {
                             processedFiles.add(virtualFile);
                             PsiFile psiFile = PsiManager.getInstance(project).findFile(virtualFile);

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt

@@ -85,6 +85,7 @@ import org.jetbrains.kotlin.codegen.extensions.ExpressionCodegenExtension
 import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
 import org.jetbrains.kotlin.config.*
 import org.jetbrains.kotlin.extensions.DeclarationAttributeAltererExtension
+import org.jetbrains.kotlin.extensions.PreprocessedVirtualFileFactoryExtension
 import org.jetbrains.kotlin.extensions.StorageComponentContainerContributor
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.load.kotlin.KotlinBinaryClassCache
@@ -160,6 +161,7 @@ class KotlinCoreEnvironment private constructor(
         PackageFragmentProviderExtension.registerExtensionPoint(project)
         StorageComponentContainerContributor.registerExtensionPoint(project)
         DeclarationAttributeAltererExtension.registerExtensionPoint(project)
+        PreprocessedVirtualFileFactoryExtension.registerExtensionPoint(project)
 
         for (registrar in configuration.getList(ComponentRegistrar.PLUGIN_COMPONENT_REGISTRARS)) {
             registrar.registerProjectComponents(project, configuration)

compiler/frontend/src/org/jetbrains/kotlin/extensions/PreprocessedVirtualFileFactoryExtension.kt

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.extensions
+
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.testFramework.LightVirtualFile
+import org.jetbrains.kotlin.utils.addToStdlib.firstNotNullResult
+
+/**
+ * The interface for the extensions that are used to substitute VirtualFile on the creation of KtFile, allows to preprocess a file before
+ * lexing and parsing
+ */
+interface PreprocessedVirtualFileFactoryExtension {
+    companion object : ProjectExtensionDescriptor<PreprocessedVirtualFileFactoryExtension>(
+            "org.jetbrains.kotlin.preprocessedVirtualFileFactoryExtension",
+            PreprocessedVirtualFileFactoryExtension::class.java
+    )
+    fun isPassThrough(): Boolean
+
+    fun createPreprocessedFile(file: VirtualFile?): VirtualFile?
+    fun createPreprocessedLightFile(file: LightVirtualFile?): LightVirtualFile?
+}
+
+class PreprocessedFileCreator(val project: Project) {
+
+    private val validExts: Array<PreprocessedVirtualFileFactoryExtension> by lazy {
+        PreprocessedVirtualFileFactoryExtension.getInstances(project).filterNot { it.isPassThrough() }.toTypedArray()
+    }
+
+    fun create(file: VirtualFile): VirtualFile = validExts.firstNotNullResult { it.createPreprocessedFile(file) } ?: file
+
+    // unused now, but could be used in the IDE at some point
+    fun createLight(file: LightVirtualFile): LightVirtualFile = validExts.firstNotNullResult { it.createPreprocessedLightFile(file) } ?: file
+}
+