azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract traversing through directories to the common code

Browse Source
This commit is contained in:
Nikolay Krasko
2020-08-10 14:56:47 +03:00
parent 8db588c7f0
commit bb0ea56d63
1 changed files with 81 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/tests/org/jetbrains/kotlin/code/CodeConformanceTest.kt
+81 -97
View File
@@ -8,8 +8,6 @@ package org.jetbrains.kotlin.code
import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.util.io.FileUtil
import com.intellij.openapi.util.io.systemIndependentPath import com.intellij.openapi.util.io.systemIndependentPath
import junit.framework.TestCase import junit.framework.TestCase
import org.jetbrains.kotlin.backend.common.pop
import org.jetbrains.kotlin.backend.common.push
import java.io.File import java.io.File
import java.util.* import java.util.*
import java.util.regex.Pattern import java.util.regex.Pattern
@@ -20,45 +18,48 @@ class CodeConformanceTest : TestCase() {
private val JAVA_FILE_PATTERN = Pattern.compile(".+\\.java") private val JAVA_FILE_PATTERN = Pattern.compile(".+\\.java")
private val SOURCES_FILE_PATTERN = Pattern.compile("(.+\\.java|.+\\.kt|.+\\.js)") private val SOURCES_FILE_PATTERN = Pattern.compile("(.+\\.java|.+\\.kt|.+\\.js)")
private const val MAX_STEPS_COUNT = 100 private const val MAX_STEPS_COUNT = 100
private val NON_SOURCE_EXCLUDED_FILES_AND_DIRS = listOf( private val nonSourcesMatcher = FileMatcher(
".git", File("."),
"build/js", listOf(
"buildSrc", ".git",
"compiler/build", "build/js",
"compiler/fir/lightTree/testData", "buildSrc",
"compiler/testData/psi/kdoc", "compiler/build",
"compiler/tests/org/jetbrains/kotlin/code/CodeConformanceTest.kt", "compiler/fir/lightTree/testData",
"compiler/util/src/org/jetbrains/kotlin/config/MavenComparableVersion.java", "compiler/testData/psi/kdoc",
"core/reflection.jvm/src/kotlin/reflect/jvm/internal/pcollections", "compiler/tests/org/jetbrains/kotlin/code/CodeConformanceTest.kt",
"dependencies", "compiler/util/src/org/jetbrains/kotlin/config/MavenComparableVersion.java",
"dependencies/protobuf/protobuf-relocated/build", "core/reflection.jvm/src/kotlin/reflect/jvm/internal/pcollections",
"dist", "dependencies",
"idea/testData/codeInsight/renderingKDoc", "dependencies/protobuf/protobuf-relocated/build",
"js/js.tests/.gradle", "dist",
"js/js.translator/qunit/qunit.js", "idea/testData/codeInsight/renderingKDoc",
"js/js.translator/testData/node_modules", "js/js.tests/.gradle",
"libraries/kotlin.test/js/it/.gradle", "js/js.translator/qunit/qunit.js",
"libraries/kotlin.test/js/it/node_modules", "js/js.translator/testData/node_modules",
"libraries/reflect/api/src/java9/java/kotlin/reflect/jvm/internal/impl", "libraries/kotlin.test/js/it/.gradle",
"libraries/reflect/build", "libraries/kotlin.test/js/it/node_modules",
"libraries/stdlib/js-ir/.gradle", "libraries/reflect/api/src/java9/java/kotlin/reflect/jvm/internal/impl",
"libraries/stdlib/js-ir/build", "libraries/reflect/build",
"libraries/stdlib/js-ir-minimal-for-test/.gradle", "libraries/stdlib/js-ir/.gradle",
"libraries/stdlib/js-ir-minimal-for-test/build", "libraries/stdlib/js-ir/build",
"libraries/stdlib/js-v1/.gradle", "libraries/stdlib/js-ir-minimal-for-test/.gradle",
"libraries/stdlib/js-v1/build", "libraries/stdlib/js-ir-minimal-for-test/build",
"libraries/tools/binary-compatibility-validator/src/main/kotlin/org.jetbrains.kotlin.tools", "libraries/stdlib/js-v1/.gradle",
"libraries/tools/kotlin-gradle-plugin-core/gradle_api_jar/build/tmp", "libraries/stdlib/js-v1/build",
"libraries/tools/kotlin-js-tests/src/test/web/qunit.js", "libraries/tools/binary-compatibility-validator/src/main/kotlin/org.jetbrains.kotlin.tools",
"libraries/tools/kotlin-maven-plugin/target", "libraries/tools/kotlin-gradle-plugin-core/gradle_api_jar/build/tmp",
"libraries/tools/kotlin-test-js-runner/.gradle", "libraries/tools/kotlin-js-tests/src/test/web/qunit.js",
"libraries/tools/kotlin-test-js-runner/lib", "libraries/tools/kotlin-maven-plugin/target",
"libraries/tools/kotlin-test-js-runner/node_modules", "libraries/tools/kotlin-test-js-runner/.gradle",
"libraries/tools/kotlin-test-nodejs-runner/.gradle", "libraries/tools/kotlin-test-js-runner/lib",
"libraries/tools/kotlin-test-nodejs-runner/node_modules", "libraries/tools/kotlin-test-js-runner/node_modules",
"libraries/tools/kotlinp/src", "libraries/tools/kotlin-test-nodejs-runner/.gradle",
"out" "libraries/tools/kotlin-test-nodejs-runner/node_modules",
).map(::File) "libraries/tools/kotlinp/src",
"out"
)
)
private val COPYRIGHT_EXCLUDED_FILES_AND_DIRS = listOf( private val COPYRIGHT_EXCLUDED_FILES_AND_DIRS = listOf(
"build", "build",
@@ -125,33 +126,25 @@ class CodeConformanceTest : TestCase() {
fun testForgottenBunchDirectivesAndFiles() { fun testForgottenBunchDirectivesAndFiles() {
val sourceBunchFilePattern = Pattern.compile("(.+\\.java|.+\\.kt|.+\\.js)(\\.\\w+)?") val sourceBunchFilePattern = Pattern.compile("(.+\\.java|.+\\.kt|.+\\.js)(\\.\\w+)?")
val root = File(".") val root = nonSourcesMatcher.root
val nonSourceMatcher = FileMatcher(root, NON_SOURCE_EXCLUDED_FILES_AND_DIRS)
val extensions = File(root, ".bunch").readLines().map { it.split("_") }.flatten().toSet() val extensions = File(root, ".bunch").readLines().map { it.split("_") }.flatten().toSet()
val failBuilder = mutableListOf<String>() val failBuilder = mutableListOf<String>()
root.walkTopDown() nonSourcesMatcher.walkTopDown(sourceBunchFilePattern).forEach { sourceFile ->
.onEnter { dir -> val matches = Regex("BUNCH (\\w+)")
!nonSourceMatcher.matchExact(dir) // Don't enter to ignored dirs .findAll(sourceFile.readText())
.map { it.groupValues[1] }
.toSet()
.filterNot { it in extensions }
for (bunch in matches) {
val filename = FileUtil.toSystemIndependentName(sourceFile.toRelativeString(root))
failBuilder.add("$filename has unregistered $bunch bunch directive")
} }
.filter { file -> !nonSourceMatcher.matchExact(file) } // filter ignored files
.filter { file -> sourceBunchFilePattern.matcher(file.name).matches() }
.filter { file -> file.isFile }
.forEach { sourceFile ->
val matches = Regex("BUNCH (\\w+)")
.findAll(sourceFile.readText())
.map { it.groupValues[1] }
.toSet()
.filterNot { it in extensions }
for (bunch in matches) {
val filename = FileUtil.toSystemIndependentName(sourceFile.toRelativeString(root))
failBuilder.add("$filename has unregistered $bunch bunch directive")
}
if (!isCorrectExtension(sourceFile.name, extensions)) { if (!isCorrectExtension(sourceFile.name, extensions)) {
val filename = FileUtil.toSystemIndependentName(sourceFile.toRelativeString(root)) val filename = FileUtil.toSystemIndependentName(sourceFile.toRelativeString(root))
failBuilder.add("$filename has unknown bunch extension") failBuilder.add("$filename has unknown bunch extension")
}
} }
}
if (failBuilder.isNotEmpty()) { if (failBuilder.isNotEmpty()) {
fail("\n" + failBuilder.joinToString("\n")) fail("\n" + failBuilder.joinToString("\n"))
@@ -208,21 +201,12 @@ class CodeConformanceTest : TestCase() {
} }
) )
val root = File(".") nonSourcesMatcher.walkTopDown(SOURCES_FILE_PATTERN).forEach { sourceFile ->
val nonSourceMatcher = FileMatcher(root, NON_SOURCE_EXCLUDED_FILES_AND_DIRS) val source = sourceFile.readText()
root.walkTopDown() for (test in tests) {
.onEnter { dir -> if (test.filter(source)) test.result.add(sourceFile)
!nonSourceMatcher.matchExact(dir) // Don't enter to ignored dirs
}
.filter { file -> !nonSourceMatcher.matchExact(file) } // filter ignored files
.filter { file -> SOURCES_FILE_PATTERN.matcher(file.name).matches() }
.filter { file -> file.isFile }
.forEach { sourceFile ->
val source = sourceFile.readText()
for (test in tests) {
if (test.filter(source)) test.result.add(sourceFile)
}
} }
}
if (tests.flatMap { it.result }.isNotEmpty()) { if (tests.flatMap { it.result }.isNotEmpty()) {
fail(buildString { fail(buildString {
@@ -263,7 +247,8 @@ class CodeConformanceTest : TestCase() {
} }
} }
private class FileMatcher(val root: File, files: Collection<File>) { private class FileMatcher(val root: File, paths: Collection<String>) {
private val files = paths.filter { !it.startsWith("*/") }.map(::File)
private val names = files.mapTo(HashSet()) { it.name } private val names = files.mapTo(HashSet()) { it.name }
private val paths = files.mapTo(HashSet()) { it.systemIndependentPath } private val paths = files.mapTo(HashSet()) { it.systemIndependentPath }
private val relativePaths = files.filter { it.isDirectory }.mapTo(HashSet()) { it.systemIndependentPath + "/" } private val relativePaths = files.filter { it.isDirectory }.mapTo(HashSet()) { it.systemIndependentPath + "/" }
@@ -279,13 +264,22 @@ class CodeConformanceTest : TestCase() {
} }
} }
private fun FileMatcher.walkTopDown(filePattern: Pattern): Sequence<File> {
return root.walkTopDown()
.onEnter { dir ->
!matchExact(dir) // Don't enter to ignored dirs
}
.filter { file -> !matchExact(file) } // filter ignored files
.filter { file -> filePattern.matcher(file.name).matches() }
.filter { file -> file.isFile }
}
fun testRepositoriesAbuse() { fun testRepositoriesAbuse() {
class RepoAllowList(val repo: String, root: File, allowList: Set<String>) { class RepoAllowList(val repo: String, root: File, allowList: Set<String>) {
val allowFiles = allowList.map(::File) val matcher = FileMatcher(root, allowList)
val matcher = FileMatcher(root, allowFiles)
} }
val root = File(".") val root = nonSourcesMatcher.root
val repoCheckers = listOf( val repoCheckers = listOf(
RepoAllowList( RepoAllowList(
@@ -307,7 +301,8 @@ class CodeConformanceTest : TestCase() {
"idea/testData/gradle/packagePrefixImport/packagePrefixNonMPP/build.gradle", "idea/testData/gradle/packagePrefixImport/packagePrefixNonMPP/build.gradle",
"idea/testData/gradle/gradleFacetImportTest/jvmImportWithCustomSourceSets_1_1_2/build.gradle", "idea/testData/gradle/gradleFacetImportTest/jvmImportWithCustomSourceSets_1_1_2/build.gradle",
"idea/testData/gradle/gradleFacetImportTest/jvmImport_1_1_2/build.gradle", "idea/testData/gradle/gradleFacetImportTest/jvmImport_1_1_2/build.gradle",
"idea/idea-gradle/tests/org/jetbrains/kotlin/idea/codeInsight/gradle/MultiplePluginVersionGradleImportingTestCase.kt" "idea/idea-gradle/tests/org/jetbrains/kotlin/idea/codeInsight/gradle/MultiplePluginVersionGradleImportingTestCase.kt",
"idea/testData/perfTest/native/_common/build.gradle.kts.header"
) )
), ),
RepoAllowList( RepoAllowList(
@@ -338,7 +333,8 @@ class CodeConformanceTest : TestCase() {
"idea/testData/gradle/configurator/configureJvmEAPWithBuildGradleKts/build.gradle.kts.after", "idea/testData/gradle/configurator/configureJvmEAPWithBuildGradleKts/build.gradle.kts.after",
"idea/testData/perfTest/native/_common/settings.gradle.kts", "idea/testData/perfTest/native/_common/settings.gradle.kts",
"kotlin-ultimate/gradle/cidrPluginTools.gradle.kts", "kotlin-ultimate/gradle/cidrPluginTools.gradle.kts",
"libraries/tools/new-project-wizard/src/org/jetbrains/kotlin/tools/projectWizard/core/service/KotlinVersionProviderService.kt" "libraries/tools/new-project-wizard/src/org/jetbrains/kotlin/tools/projectWizard/core/service/KotlinVersionProviderService.kt",
"idea/testData/perfTest/native/_common/build.gradle.kts.header"
) )
) )
) )
@@ -346,20 +342,8 @@ class CodeConformanceTest : TestCase() {
data class RepoOccurance(val repo: String, val file: File) data class RepoOccurance(val repo: String, val file: File)
data class RepoOccurrences(val repo: String, val files: Collection<File>) data class RepoOccurrences(val repo: String, val files: Collection<File>)
val extensions = hashSetOf("java", "kt", "gradle", "kts", "xml", "after") val extensionsPattern = Pattern.compile(".+\\.(java|kt|gradle|kts|xml)(\\.\\w+)?")
val nonSourceMatcher = FileMatcher(root, NON_SOURCE_EXCLUDED_FILES_AND_DIRS) val repoOccurrences: List<RepoOccurrences> = nonSourcesMatcher.walkTopDown(extensionsPattern)
val traceStack = mutableListOf<Long>()
val repoOccurrences: List<RepoOccurrences> = root.walkTopDown()
.onEnter { dir ->
traceStack.push(System.nanoTime())
!nonSourceMatcher.matchExact(dir)
} // don't visit dirs
.onLeave { dir ->
val start = traceStack.pop()
println("${dir.path} - ${(System.nanoTime() - start) / 1000}")
}
.filter { file -> file.extension in extensions && file.isFile }
.filter { file -> !nonSourceMatcher.matchExact(file) } // filter ignored files
.flatMap { file -> .flatMap { file ->
val checkers = repoCheckers.filter { checker -> val checkers = repoCheckers.filter { checker ->
!checker.matcher.matchWithContains(file) !checker.matcher.matchWithContains(file)