[KT-39120] Add "-fmodules" argument support to Cinterop

Merge-request: KT-MR-6921
Merged-by: Vladimir Sukharev <Vladimir.Sukharev@jetbrains.com>

kotlin-native/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt

@@ -151,7 +151,7 @@ public open class NativeIndexImpl(val library: NativeLibrary, val verbose: Boole
 
     override lateinit var includedHeaders: List<HeaderId>
 
-    private fun log(message: String) {
+    internal fun log(message: String) {
         if (verbose) {
             println(message)
         }
@@ -1187,67 +1187,85 @@ fun buildNativeIndexImpl(index: NativeIndexImpl): IndexerResult {
 }
 
 private fun indexDeclarations(nativeIndex: NativeIndexImpl): CompilationWithPCH {
-    withIndex { index ->
+    // Below, declarations from PCH should be excluded to restrict `visitChildren` to visit local declarations only
+    withIndex(excludeDeclarationsFromPCH = true) { index ->
+        val errors = mutableListOf<Diagnostic>()
         val translationUnit = nativeIndex.library.copyWithArgsForPCH().parse(
                 index,
-                options = CXTranslationUnit_DetailedPreprocessingRecord or CXTranslationUnit_ForSerialization
+                options = CXTranslationUnit_DetailedPreprocessingRecord or CXTranslationUnit_ForSerialization,
+                diagnosticHandler = { if (it.isError()) errors.add(it) }
         )
         try {
+            if (errors.isNotEmpty()) {
+                error(errors.take(10).joinToString("\n") { it.format })
+            }
             translationUnit.ensureNoCompileErrors()
 
             val compilation = nativeIndex.library.withPrecompiledHeader(translationUnit)
 
-            val headers = getFilteredHeaders(nativeIndex, index, translationUnit)
+            UnitsHolder(index).use { unitsHolder ->
+                val (headers, ownTranslationUnits) = getHeadersAndUnits(nativeIndex.library, index, translationUnit, unitsHolder)
+                val ownHeaders = headers.ownHeaders
+                val headersCanonicalPaths = ownHeaders.map { it?.canonicalPath }.toSet()
 
-            nativeIndex.includedHeaders = headers.map {
-                nativeIndex.getHeaderId(it)
-            }
+                val unitsToProcess = (ownTranslationUnits + setOf(translationUnit)).toList()
 
-            indexTranslationUnit(index, translationUnit, 0, object : Indexer {
-                override fun indexDeclaration(info: CXIdxDeclInfo) {
-                    val file = memScoped {
-                        val fileVar = alloc<CXFileVar>()
-                        clang_indexLoc_getFileLocation(info.loc.readValue(), null, fileVar.ptr, null, null, null)
-                        fileVar.value
-                    }
-
-                    if (file in headers) {
-                        nativeIndex.indexDeclaration(info)
-                    }
+                nativeIndex.includedHeaders = ownHeaders.map {
+                    nativeIndex.getHeaderId(it)
                 }
-            })
 
-            if (nativeIndex.library.language == Language.CPP) {
-                visitChildren(clang_getTranslationUnitCursor(translationUnit)) { cursor, _ ->
-                    if (getContainingFile(cursor) in headers) {
-                        nativeIndex.indexCxxDeclaration(cursor)
-                    }
-                    CXChildVisitResult.CXChildVisit_Continue
-                }
-            }
+                unitsToProcess.forEach {
+                    indexTranslationUnit(index, it, 0, object : Indexer {
+                        override fun indexDeclaration(info: CXIdxDeclInfo) {
+                            val file = memScoped {
+                                val fileVar = alloc<CXFileVar>()
+                                clang_indexLoc_getFileLocation(info.loc.readValue(), null, fileVar.ptr, null, null, null)
+                                fileVar.value
+                            }
 
-            visitChildren(clang_getTranslationUnitCursor(translationUnit)) { cursor, _ ->
-                val file = getContainingFile(cursor)
-                if (file in headers && nativeIndex.library.includesDeclaration(cursor)) {
-                    when (cursor.kind) {
-                        CXCursorKind.CXCursor_ObjCInterfaceDecl -> nativeIndex.indexObjCClass(cursor)
-                        CXCursorKind.CXCursor_ObjCProtocolDecl -> nativeIndex.indexObjCProtocol(cursor)
-                        CXCursorKind.CXCursor_ObjCCategoryDecl -> {
-                            // This fixes https://youtrack.jetbrains.com/issue/KT-49455, which effectively seems to be a bug in libclang:
-                            // the libclang indexer doesn't properly index categories with
-                            // `__attribute__((external_source_symbol(language="Swift",...)))`.
-                            // As a workaround, additionally enumerate all the categories explicitly.
-                            nativeIndex.indexObjCCategory(cursor)
+                            if (file?.canonicalPath in headersCanonicalPaths) {
+                                nativeIndex.indexDeclaration(info)
+                            }
+                        }
+                    })
+                }
+
+                if (nativeIndex.library.language == Language.CPP) {
+                    unitsToProcess.forEach {
+                        visitChildren(clang_getTranslationUnitCursor(it)) { cursor, _ ->
+                            if (getContainingFile(cursor) in ownHeaders) {
+                                nativeIndex.indexCxxDeclaration(cursor)
+                            }
+                            CXChildVisitResult.CXChildVisit_Continue
                         }
-                        else -> {}
                     }
                 }
-                CXChildVisitResult.CXChildVisit_Continue
+
+                unitsToProcess.forEach {
+                    visitChildren(clang_getTranslationUnitCursor(it)) { cursor, _ ->
+                        val file = getContainingFile(cursor)
+                        if (file in ownHeaders && nativeIndex.library.includesDeclaration(cursor)) {
+                            when (cursor.kind) {
+                                CXCursorKind.CXCursor_ObjCInterfaceDecl -> nativeIndex.indexObjCClass(cursor)
+                                CXCursorKind.CXCursor_ObjCProtocolDecl -> nativeIndex.indexObjCProtocol(cursor)
+                                CXCursorKind.CXCursor_ObjCCategoryDecl -> {
+                                    // This fixes https://youtrack.jetbrains.com/issue/KT-49455, which effectively seems to be a bug in libclang:
+                                    // the libclang indexer doesn't properly index categories with
+                                    // `__attribute__((external_source_symbol(language="Swift",...)))`.
+                                    // As a workaround, additionally enumerate all the categories explicitly.
+                                    nativeIndex.indexObjCCategory(cursor)
+                                }
+
+                                else -> {}
+                            }
+                        }
+                        CXChildVisitResult.CXChildVisit_Continue
+                    }
+                }
+
+                findMacros(nativeIndex, compilation, unitsToProcess, ownHeaders)
+                return compilation
             }
-
-            findMacros(nativeIndex, compilation, translationUnit, headers)
-
-            return compilation
         } finally {
             clang_disposeTranslationUnit(translationUnit)
         }

kotlin-native/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/MacroConstants.kt

@@ -20,16 +20,18 @@ import clang.*
 import kotlinx.cinterop.*
 import java.io.File
 
+val predefinedMacros = setOf("__DATE__", "__TIME__", "__TIMESTAMP__", "__FILE__", "__FILE_NAME__", "__BASE_FILE__", "__LINE__")
+
 /**
  * Finds all "macro constants" and registers them as [NativeIndex.constants] in given index.
  */
 internal fun findMacros(
         nativeIndex: NativeIndexImpl,
         compilation: CompilationWithPCH,
-        translationUnit: CXTranslationUnit,
+        translationUnits: List<CXTranslationUnit>,
         headers: Set<CXFile?>
 ) {
-    val names = collectMacroNames(nativeIndex, translationUnit, headers)
+    val names = collectMacroNames(nativeIndex, translationUnits, headers)
     // TODO: apply user-defined filters.
     val macros = expandMacros(compilation, names, typeConverter = { nativeIndex.convertType(it) })
 
@@ -60,17 +62,10 @@ private fun expandMacros(
         // or function-like construction (e.g. #define FOO throw()) but such a function is undeclared.
         compilerArgs += "-Werror=implicit-function-declaration"
 
-        // Some predefined macros expand to contextual values that won't make sense to expose in Kotlin properties.
-        // We instead redefined them to string values of the macro name.
-        compilerArgs += "-Wno-builtin-macro-redefined"
-        val predefinedMacros = listOf("__DATE__", "__TIME__", "__TIMESTAMP__", "__FILE__", "__FILE_NAME__", "__BASE_FILE__", "__LINE__")
-        predefinedMacros.forEach {
-            compilerArgs += "-D${it}=\"${it}\""
-        }
         // Ensure libclang reports all errors:
         compilerArgs += "-ferror-limit=0"
 
-        val translationUnit = parseTranslationUnit(index, sourceFile, compilerArgs, options = 0)
+        val translationUnit = parseTranslationUnit(index, sourceFile, compilerArgs, options = CXTranslationUnit_DetailedPreprocessingRecord)
         try {
             val nameToMacroDef = mutableMapOf<String, MacroDef>()
             val unprocessedMacros = names.toMutableList()
@@ -188,7 +183,7 @@ private fun reparseWithCodeSnippets(library: CompilationWithPCH,
             codeSnippetLines.forEach { writer.appendLine(it) }
         }
     }
-    clang_reparseTranslationUnit(translationUnit, 0, null, 0)
+    clang_reparseTranslationUnit(translationUnit, 0, null, CXTranslationUnit_DetailedPreprocessingRecord)
 }
 
 /**
@@ -291,29 +286,30 @@ enum class VisitorState {
     EXPECT_END, INVALID
 }
 
-private fun collectMacroNames(nativeIndex: NativeIndexImpl, translationUnit: CXTranslationUnit, headers: Set<CXFile?>): List<String> {
+private fun collectMacroNames(nativeIndex: NativeIndexImpl, translationUnits: List<CXTranslationUnit>, headers: Set<CXFile?>): List<String> {
     val result = mutableSetOf<String>()
 
-    visitChildren(translationUnit) { cursor, _ ->
-        val file = memScoped {
-            val fileVar = alloc<CXFileVar>()
-            clang_getFileLocation(clang_getCursorLocation(cursor), fileVar.ptr, null, null, null)
-            fileVar.value
-        }
+    translationUnits.forEach {
+        visitChildren(it) { cursor, _ ->
+            val file = memScoped {
+                val fileVar = alloc<CXFileVar>()
+                clang_getFileLocation(clang_getCursorLocation(cursor), fileVar.ptr, null, null, null)
+                fileVar.value
+            }
 
-        if (cursor.kind == CXCursorKind.CXCursor_MacroDefinition &&
-                nativeIndex.library.includesDeclaration(cursor) &&
-                file != null && // Builtin macros mostly seem to be useless.
-                file in headers &&
-                canMacroBeConstant(cursor))
-        {
-            val spelling = getCursorSpelling(cursor)
-            result.add(spelling)
+            if (cursor.kind == CXCursorKind.CXCursor_MacroDefinition &&
+                    nativeIndex.library.includesDeclaration(cursor) &&
+                    file != null && // Builtin macros mostly seem to be useless.
+                    file in headers &&
+                    canMacroBeConstant(cursor)) {
+                val spelling = getCursorSpelling(cursor)
+                result.add(spelling)
+            }
+            CXChildVisitResult.CXChildVisit_Continue
         }
-        CXChildVisitResult.CXChildVisit_Continue
     }
 
-    return result.toList()
+    return result.filterNot { predefinedMacros.contains(it) }.toList()
 }
 
 private fun canMacroBeConstant(cursor: CValue<CXCursor>): Boolean {

kotlin-native/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/ModuleSupport.kt

@@ -4,12 +4,12 @@ import clang.*
 import kotlinx.cinterop.*
 import java.nio.file.Files
 
-data class ModulesInfo(val topLevelHeaders: List<String>, val ownHeaders: Set<String>)
+data class ModulesInfo(val topLevelHeaders: List<String>, val ownHeaders: Set<String>, val modules: List<String>)
 
 fun getModulesInfo(compilation: Compilation, modules: List<String>): ModulesInfo {
-    if (modules.isEmpty()) return ModulesInfo(emptyList(), emptySet())
+    if (modules.isEmpty()) return ModulesInfo(emptyList(), emptySet(), emptyList())
 
-    withIndex { index ->
+    withIndex(excludeDeclarationsFromPCH = false) { index ->
         ModularCompilation(compilation).use {
             val modulesASTFiles = getModulesASTFiles(index, it, modules)
             return buildModulesInfo(index, modules, modulesASTFiles)
@@ -30,7 +30,7 @@ private fun buildModulesInfo(index: CXIndex, modules: List<String>, modulesASTFi
         }
     }
 
-    return ModulesInfo(topLevelHeaders.toList(), ownHeaders)
+    return ModulesInfo(topLevelHeaders.toList(), ownHeaders, modules)
 }
 
 internal open class ModularCompilation(compilation: Compilation): Compilation by compilation, Disposable {

kotlin-native/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt

@@ -48,7 +48,7 @@ sealed class NativeLibraryHeaderFilter {
             val excludeDepdendentModules: Boolean
     ) : NativeLibraryHeaderFilter()
 
-    class Predefined(val headers: Set<String>) : NativeLibraryHeaderFilter()
+    class Predefined(val headers: Set<String>, val modules: List<String>) : NativeLibraryHeaderFilter()
 }
 
 interface Compilation {

kotlin-native/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt

@@ -25,6 +25,7 @@ import java.nio.file.Path
 import java.nio.file.Paths
 import java.security.DigestInputStream
 import java.security.MessageDigest
+import java.util.*
 import java.util.concurrent.ConcurrentHashMap
 
 val CValue<CXType>.kind: CXTypeKind get() = this.useContents { kind }
@@ -105,7 +106,7 @@ internal fun CValue<CXType>.getSize(): Long {
 }
 
 internal inline fun <R> withIndex(
-        excludeDeclarationsFromPCH: Boolean = false,
+        excludeDeclarationsFromPCH: Boolean, // disables visitChildren to visit declarations from imported translation units
         displayDiagnostics: Boolean = false,
         block: (index: CXIndex) -> R
 ): R {
@@ -404,8 +405,8 @@ data class CompilationImpl(
  *
  * @return the library which includes the precompiled header instead of original ones.
  */
-fun Compilation.precompileHeaders(): CompilationWithPCH = withIndex { index ->
-    val options = CXTranslationUnit_ForSerialization
+fun Compilation.precompileHeaders(): CompilationWithPCH = withIndex(excludeDeclarationsFromPCH = false) { index ->
+    val options = CXTranslationUnit_ForSerialization or CXTranslationUnit_DetailedPreprocessingRecord
     val translationUnit = copyWithArgsForPCH().parse(index, options)
     try {
         translationUnit.ensureNoCompileErrors()
@@ -471,7 +472,7 @@ fun List<List<String>>.mapFragmentIsCompilable(originalLibrary: CompilationWithP
 
     withIndex(excludeDeclarationsFromPCH = true) { index ->
         val sourceFile = library.createTempSource()
-        val translationUnit = parseTranslationUnit(index, sourceFile, library.compilerArgs, options = 0)
+        val translationUnit = parseTranslationUnit(index, sourceFile, library.compilerArgs, options = CXTranslationUnit_DetailedPreprocessingRecord)
         try {
             translationUnit.ensureNoCompileErrors()
             while (fragmentsToCheck.isNotEmpty()) {
@@ -486,7 +487,7 @@ fun List<List<String>>.mapFragmentIsCompilable(originalLibrary: CompilationWithP
                     }
                 }
 
-                clang_reparseTranslationUnit(translationUnit, 0, null, 0)
+                clang_reparseTranslationUnit(translationUnit, 0, null, CXTranslationUnit_DetailedPreprocessingRecord)
                 val errorLineNumbers = translationUnit.getErrorLineNumbers().toSet()
 
                 // Retain only those fragments that contain compilation error locations:
@@ -653,19 +654,16 @@ internal fun getHeaderId(library: NativeLibrary, header: CXFile?): HeaderId {
     return library.headerToIdMapper.getHeaderId(filePath)
 }
 
-internal fun getFilteredHeaders(
-        nativeIndex: NativeIndexImpl,
-        index: CXIndex,
-        translationUnit: CXTranslationUnit
-): Set<CXFile?> = getHeaders(nativeIndex.library, index, translationUnit).ownHeaders
-
 class NativeLibraryHeaders<Header>(val ownHeaders: Set<Header>, val importedHeaders: Set<Header>)
+data class NativeLibraryHeadersAndUnits(val headers: NativeLibraryHeaders<CXFile?>, val ownTranslationUnits: Set<CXTranslationUnit>)
 
-internal fun getHeaders(
+internal fun getHeadersAndUnits(
         library: NativeLibrary,
         index: CXIndex,
-        translationUnit: CXTranslationUnit
-): NativeLibraryHeaders<CXFile?> {
+        translationUnit: CXTranslationUnit,
+        unitsHolder: UnitsHolder
+): NativeLibraryHeadersAndUnits {
+    val ownTranslationUnits = mutableSetOf<CXTranslationUnit>()
     val ownHeaders = mutableSetOf<CXFile?>()
     val allHeaders = mutableSetOf<CXFile?>(null)
 
@@ -673,15 +671,31 @@ internal fun getHeaders(
 
     when (filter) {
         is NativeLibraryHeaderFilter.NameBased ->
-            filterHeadersByName(library, filter, index, translationUnit, ownHeaders, allHeaders)
+            filterHeadersByName(library, filter, index, translationUnit, ownTranslationUnits, ownHeaders, allHeaders, unitsHolder)
 
         is NativeLibraryHeaderFilter.Predefined ->
-            filterHeadersByPredefined(filter, index, translationUnit, ownHeaders, allHeaders)
+            filterHeadersByPredefined(filter, index, translationUnit, ownTranslationUnits, ownHeaders, allHeaders, unitsHolder)
     }
 
     ownHeaders.removeAll { library.headerExclusionPolicy.excludeAll(getHeaderId(library, it)) }
 
-    return NativeLibraryHeaders(ownHeaders, allHeaders - ownHeaders)
+    return NativeLibraryHeadersAndUnits(NativeLibraryHeaders(ownHeaders, allHeaders - ownHeaders), ownTranslationUnits)
+}
+
+class UnitsHolder(val index: CXIndex) : Disposable {
+    private val unitByBinaryFile = mutableMapOf<String, CXTranslationUnit>()
+
+    internal fun load(info: CXIdxImportedASTFileInfo): CXTranslationUnit {
+        val canonicalPath: String = info.file!!.canonicalPath
+        return unitByBinaryFile.getOrPut(canonicalPath) {
+            clang_createTranslationUnit(index, canonicalPath)!!
+        }
+    }
+
+    override fun dispose() {
+        unitByBinaryFile.values.forEach { clang_disposeTranslationUnit(it) }
+        unitByBinaryFile.clear()
+    }
 }
 
 private fun filterHeadersByName(
@@ -689,57 +703,78 @@ private fun filterHeadersByName(
         filter: NativeLibraryHeaderFilter.NameBased,
         index: CXIndex,
         translationUnit: CXTranslationUnit,
+        ownTranslationUnits: MutableSet<CXTranslationUnit>,
         ownHeaders: MutableSet<CXFile?>,
-        allHeaders: MutableSet<CXFile?>
+        allHeaders: MutableSet<CXFile?>,
+        unitsHolder: UnitsHolder
 ) {
-    val topLevelFiles = mutableListOf<CXFile>()
+    val topLevelFiles = mutableSetOf<CXFile>()
     var mainFile: CXFile? = null
+    val translationUnits = mutableListOf(translationUnit)
 
-    indexTranslationUnit(index, translationUnit, 0, object : Indexer {
-        val headerToName = mutableMapOf<CXFile, String>()
-        // The *name* of the header here is the path relative to the include path element., e.g. `curl/curl.h`.
+    // The *name* of the header here is the path relative to the include path element., e.g. `curl/curl.h`.
+    val headerToName = mutableMapOf<String, String>()
 
-        override fun enteredMainFile(file: CXFile) {
-            mainFile = file
-            allHeaders += file
-        }
+    var curUnitIndex = 0
+    while (curUnitIndex < translationUnits.size) {
+        val curUnit = translationUnits[curUnitIndex++]
 
-        override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
-            val includeLocation = clang_indexLoc_getCXSourceLocation(info.hashLoc.readValue())
-            val file = info.file!!
-
-            allHeaders += file
-
-            if (clang_Location_isFromMainFile(includeLocation) != 0) {
-                topLevelFiles.add(file)
+        indexTranslationUnit(index, curUnit, 0, object : Indexer {
+            override fun enteredMainFile(file: CXFile) {
+                mainFile = file
+                allHeaders += file
             }
 
-            val name = info.filename!!.toKString()
-            val headerName = if (info.isAngled != 0) {
-                // If the header is included with `#include <$name>`, then `name` is probably
-                // the path relative to the include path element.
-                name
-            } else {
-                // If it is included with `#include "$name"`, then `name` can also be the path relative to the includer.
-                val includerFile = includeLocation.getContainingFile()!!
-                val includerName = headerToName[includerFile] ?: ""
-                val includerPath = includerFile.path
+            override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
+                val includeLocation = clang_indexLoc_getCXSourceLocation(info.hashLoc.readValue())
+                val file = info.file!!
 
-                if (clang_getFile(translationUnit, Paths.get(includerPath).resolveSibling(name).toString()) == file) {
-                    // included file is accessible from the includer by `name` used as relative path, so
-                    // `name` seems to be relative to the includer:
-                    Paths.get(includerName).resolveSibling(name).normalize().toString()
-                } else {
+                allHeaders += file
+
+                if (clang_Location_isFromMainFile(includeLocation) != 0) {
+                    topLevelFiles.add(file)
+                }
+
+                val name = info.filename!!.toKString()
+                val headerName = if (info.isAngled != 0) {
+                    // If the header is included with `#include <$name>`, then `name` is probably
+                    // the path relative to the include path element.
                     name
+                } else {
+                    // If it is included with `#include "$name"`, then `name` can also be the path relative to the includer.
+                    // Warning: containingFile is null when one module imports another via AST file
+                    val includerFile = includeLocation.getContainingFile()!!
+                    val includerName = headerToName[includerFile.canonicalPath] ?: ""
+                    val includerPath = includerFile.path
+
+                    val resolvedSibling = Paths.get(includerPath).resolveSibling(name).toString()
+                    if (clang_getFile(curUnit, resolvedSibling) == file) {
+                        // included file is accessible from the includer by `name` used as relative path, so
+                        // `name` seems to be relative to the includer:
+                        Paths.get(includerName).resolveSibling(name).normalize().toString()
+                    } else {
+                        name
+                    }
+                }
+
+                headerToName[file.canonicalPath] = headerName
+
+                if (!filter.policy.excludeUnused(headerName)) {
+                    ownHeaders.add(file)
+                    ownTranslationUnits += curUnit
                 }
             }
 
-            headerToName[file] = headerName
-            if (!filter.policy.excludeUnused(headerName)) {
-                ownHeaders.add(file)
+            override fun importedASTFile(info: CXIdxImportedASTFileInfo) {
+                unitsHolder.load(info).also { unit ->
+                    if (!translationUnits.contains(unit)) {
+                        translationUnits.add(unit)
+                        ownTranslationUnits += unit
+                    }
+                }
             }
-        }
-    })
+        })
+    }
 
     if (filter.excludeDepdendentModules) {
         ModulesMap(compilation, translationUnit).use { modulesMap ->
@@ -765,35 +800,70 @@ private fun filterHeadersByPredefined(
         filter: NativeLibraryHeaderFilter.Predefined,
         index: CXIndex,
         translationUnit: CXTranslationUnit,
+        ownTranslationUnits: MutableSet<CXTranslationUnit>,
         ownHeaders: MutableSet<CXFile?>,
-        allHeaders: MutableSet<CXFile?>
+        allHeaders: MutableSet<CXFile?>,
+        unitsHolder: UnitsHolder
 ) {
+    val translationUnits = mutableListOf(translationUnit)
     // Note: suboptimal but simple.
-    indexTranslationUnit(index, translationUnit, 0, object : Indexer {
-        override fun enteredMainFile(file: CXFile) {
-            ownHeaders += file
-            allHeaders += file
-        }
-
-        override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
-            val file = info.file
-            allHeaders += file
-            if (file?.canonicalPath in filter.headers) {
+    var curUnitIndex = 0
+    while (curUnitIndex < translationUnits.size) {
+        indexTranslationUnit(index, translationUnits[curUnitIndex++], 0, object : Indexer {
+            override fun enteredMainFile(file: CXFile) {
                 ownHeaders += file
+                allHeaders += file
             }
-        }
-    })
+
+            override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
+                val file = info.file
+                allHeaders += file
+                if (file?.canonicalPath in filter.headers) {
+                    ownHeaders += file
+                }
+            }
+
+            override fun importedASTFile(info: CXIdxImportedASTFileInfo) {
+                unitsHolder.load(info).also { unit ->
+                    if (!translationUnits.contains(unit)) {
+                        translationUnits.add(unit)
+
+                        // `info.module` might point to a submodule having name of a child header, not an actual name of a framework.
+                        // Actual module name could be found at top of the parent chain
+                        val topParentModuleName = getTopParentModule(info)?.name
+                        if (filter.modules.contains(topParentModuleName)) {
+                            ownTranslationUnits += unit
+                        }
+                    }
+                }
+            }
+
+            /**
+             * Follows parent links and returns name of the topmost parent module.
+             */
+            private fun getTopParentModule(info: CXIdxImportedASTFileInfo): CXModule? {
+                var parent = info.module
+                var module: CXModule?
+                do {
+                    module = parent
+                    parent = clang_Module_getParent(module)
+                } while (parent != null)
+                return module
+            }
+        })
+    }
 }
 
 fun NativeLibrary.getHeaderPaths(): NativeLibraryHeaders<String> {
-    withIndex { index ->
+    withIndex(excludeDeclarationsFromPCH = false) { index ->
         val translationUnit =
                 this.parse(index, options = CXTranslationUnit_DetailedPreprocessingRecord).ensureNoCompileErrors()
         try {
+            val (headers, _) = UnitsHolder(index).use { unitsHolder ->
+                getHeadersAndUnits(this, index, translationUnit, unitsHolder)
+            }
 
             fun getPath(file: CXFile?) = if (file == null) "<builtins>" else file.canonicalPath
-
-            val headers = getHeaders(this, index, translationUnit)
             return NativeLibraryHeaders(
                     headers.ownHeaders.map(::getPath).toSet(),
                     headers.importedHeaders.map(::getPath).toSet()
@@ -837,6 +907,7 @@ internal fun getContainingFile(cursor: CValue<CXCursor>): CXFile? {
 }
 
 internal val CXFile.path: String get() = clang_getFileName(this).convertAndDispose()
+internal val CXModule.name: String get() = clang_Module_getName(this).convertAndDispose()
 
 // TODO: this map doesn't get cleaned up but adds quite significant performance improvement.
 private val canonicalPaths = ConcurrentHashMap<String, String>()
@@ -906,11 +977,11 @@ fun Type.canonicalIsPointerToChar(): Boolean {
     return unwrappedType is PointerType && unwrappedType.pointeeType.unwrapTypedefs() == CharType
 }
 
-internal interface Disposable {
+interface Disposable {
     fun dispose()
 }
 
-internal inline fun <T : Disposable, R> T.use(block: (T) -> R): R = try {
+inline fun <T : Disposable, R> T.use(block: (T) -> R): R = try {
     block(this)
 } finally {
     this.dispose()

kotlin-native/Interop/Indexer/src/test/kotlin/org/jetbrains/kotlin/native/interop/indexer/WorkaroundTests.kt

@@ -33,7 +33,7 @@ class WorkaroundTests : IndexerTests() {
                 compilerArgs = defaultCompilerArgs(language),
                 language = language
         )
-        withIndex { index ->
+        withIndex(excludeDeclarationsFromPCH = false) { index ->
             val translationUnit = compilation.parse(
                     index,
                     options = CXTranslationUnit_DetailedPreprocessingRecord,

kotlin-native/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/StubIrBuilder.kt

@@ -335,7 +335,8 @@ class StubIrBuilder(private val context: StubIrContext) {
         nativeIndex.enums.forEach { generateStubsForEnum(it) }
         nativeIndex.functions.filter { it.name !in excludedFunctions }.forEach { generateStubsForFunction(it) }
         nativeIndex.typedefs.forEach { generateStubsForTypedef(it) }
-        nativeIndex.globals.filter { it.name !in excludedFunctions }.forEach { generateStubsForGlobal(it) }
+        // globals are sorted, so its numbering is stable and thus testable with golden data
+        nativeIndex.globals.filter { it.name !in excludedFunctions }.sortedBy { it.name }.forEach { generateStubsForGlobal(it) }
         nativeIndex.macroConstants.filter { it.name !in excludedMacros }.forEach { generateStubsForMacroConstant(it) }
         nativeIndex.wrappedMacros.filter { it.name !in excludedMacros }.forEach { generateStubsForWrappedMacro(it) }
 

kotlin-native/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt

@@ -68,21 +68,36 @@ fun main(args: Array<String>) {
     processCLibSafe(flavorName, arguments, InternalInteropOptions(arguments.generated, arguments.natives), runFromDaemon = false)
 }
 
-fun interop(
-        flavor: String, args: Array<String>,
-        additionalArgs: InternalInteropOptions,
-        runFromDaemon: Boolean
-): Array<String>? = when (flavor) {
-    "jvm", "native" -> {
-        val cinteropArguments = CInteropArguments()
-        cinteropArguments.argParser.parse(args)
-        val platform = KotlinPlatform.values().single { it.name.equals(flavor, ignoreCase = true) }
-        processCLibSafe(platform, cinteropArguments, additionalArgs, runFromDaemon)
+class Interop {
+    /**
+     * invoked via reflection from new test system: CompilationToolCallKt.invokeCInterop(),
+     * `interop()` has issues to be invoked directly due to NoSuchMethodError, caused by presence of InternalInteropOptions argtype:
+     * java.lang.IllegalArgumentException: argument type mismatch
+     */
+    fun interopViaReflection(
+            flavor: String, args: Array<String>,
+            runFromDaemon: Boolean,
+            generated: String, natives: String, manifest: String? = null, cstubsName: String? = null
+    ): Array<String>? {
+        val internalInteropOptions = InternalInteropOptions(generated, natives, manifest, cstubsName)
+        return interop(flavor, args, internalInteropOptions, runFromDaemon)
     }
-    "wasm" -> processIdlLib(args, additionalArgs)
-    else -> error("Unexpected flavor")
-}
 
+    fun interop(
+            flavor: String, args: Array<String>,
+            additionalArgs: InternalInteropOptions,
+            runFromDaemon: Boolean
+    ): Array<String>? = when (flavor) {
+        "jvm", "native" -> {
+            val cinteropArguments = CInteropArguments()
+            cinteropArguments.argParser.parse(args)
+            val platform = KotlinPlatform.values().single { it.name.equals(flavor, ignoreCase = true) }
+            processCLibSafe(platform, cinteropArguments, additionalArgs, runFromDaemon)
+        }
+        "wasm" -> processIdlLib(args, additionalArgs)
+        else -> error("Unexpected flavor")
+    }
+}
 // Options, whose values are space-separated and can be escaped.
 val escapedOptions = setOf("-compilerOpts", "-linkerOpts", "-compiler-options", "-linker-options")
 
@@ -390,7 +405,7 @@ private fun processCLib(flavor: KotlinPlatform, cinteropArguments: CInteropArgum
             // Note that the output bitcode contains the source file path, which can lead to non-deterministc builds (see KT-54284).
             // The source file is passed in via stdin to ensure the output library is deterministic.
             val compilerCmd = arrayOf(compiler, *compilerArgs,
-                    "-emit-llvm", "-x", library.language.clangLanguageName, "-c", "-", "-o", outLib.absolutePath)
+                    "-emit-llvm", "-x", library.language.clangLanguageName, "-c", "-", "-o", outLib.absolutePath, "-Xclang", "-detailed-preprocessing-record")
             runCmd(compilerCmd, verbose, redirectInputFile = File(outCFile.absolutePath))
             outLib.absolutePath
         }
@@ -496,11 +511,19 @@ internal fun buildNativeLibrary(
         addAll(tool.getDefaultCompilerOptsForLanguage(language))
         addAll(additionalCompilerOpts)
         addAll(getCompilerFlagsForVfsOverlay(arguments.headerFilterPrefix.toTypedArray(), def))
+        add("-Wno-builtin-macro-redefined") // to suppress warning from predefinedMacrosRedefinitions(see below)
+    }
+
+    // Expanding macros such as __FILE__ or __TIME__ exposes arbitrary generated filenames and timestamps from the compiler pipeline
+    // which are not useful for interop though makes the klib generation non-deterministic. See KT-54284
+    // This macro redefinition just maps to their name in the properties available from Kotlin.
+    val predefinedMacrosRedefinitions = predefinedMacros.map {
+        "#define $it \"$it\""
     }
 
     val compilation = CompilationImpl(
             includes = headerFiles,
-            additionalPreambleLines = def.defHeaderLines,
+            additionalPreambleLines = def.defHeaderLines + predefinedMacrosRedefinitions,
             compilerArgs = defaultCompilerArgs(language) + compilerOpts + tool.platformCompilerOpts,
             language = language
     )
@@ -511,6 +534,7 @@ internal fun buildNativeLibrary(
     val modules = def.config.modules
 
     if (modules.isEmpty()) {
+        require(headerFiles.isEmpty() || !compilation.compilerArgs.contains("-fmodules")) { "cinterop doesn't support having headers in -fmodules mode" }
         val excludeDependentModules = def.config.excludeDependentModules
 
         val headerFilterGlobs = def.config.headerFilter
@@ -526,7 +550,7 @@ internal fun buildNativeLibrary(
 
         val modulesInfo = getModulesInfo(compilation, modules)
 
-        headerFilter = NativeLibraryHeaderFilter.Predefined(modulesInfo.ownHeaders)
+        headerFilter = NativeLibraryHeaderFilter.Predefined(modulesInfo.ownHeaders, modulesInfo.modules)
         includes = modulesInfo.topLevelHeaders
     }
 

kotlin-native/Interop/StubGenerator/src/test/kotlin/org/jetbrains/kotlin/native/interop/gen/InteropTestsBase.kt

@@ -13,7 +13,6 @@ import org.jetbrains.kotlin.native.interop.gen.jvm.KotlinPlatform
 import org.jetbrains.kotlin.native.interop.gen.jvm.buildNativeLibrary
 import org.jetbrains.kotlin.native.interop.gen.jvm.prepareTool
 import org.jetbrains.kotlin.native.interop.indexer.NativeLibrary
-import org.jetbrains.kotlin.native.interop.indexer.getHeaderPaths
 import org.jetbrains.kotlin.native.interop.tool.CInteropArguments
 import kotlin.test.*
 import java.io.File