From dd0e04edd5c259bd0f8b7a46e3eb4a1d058ae6b1 Mon Sep 17 00:00:00 2001 From: Ilya Gorbunov Date: Fri, 10 Nov 2017 05:42:02 +0300 Subject: [PATCH] Switch production generated stdlib sources to the new DSL and remove old DSL --- .../src/generators/GenerateStandardLib.kt | 133 ++--- .../src/templates/dsl/Writers.kt | 6 +- .../src/templates/engine/Engine.kt | 462 ------------------ 3 files changed, 39 insertions(+), 562 deletions(-) delete mode 100644 libraries/tools/kotlin-stdlib-gen/src/templates/engine/Engine.kt diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/GenerateStandardLib.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/GenerateStandardLib.kt index d702b912abd..16e0a360b74 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/GenerateStandardLib.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/GenerateStandardLib.kt @@ -2,12 +2,6 @@ package generators import java.io.* import templates.* -import templates.Family.* - -val COMMON_AUTOGENERATED_WARNING: String = """// -// NOTE THIS FILE IS AUTO-GENERATED by the GenerateStandardLib.kt -// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib -//""" /** * Generates methods in the standard library which are mostly identical @@ -17,105 +11,46 @@ val COMMON_AUTOGENERATED_WARNING: String = """// * at runtime. */ fun main(args: Array) { + val templateGroups = sequenceOf( +// Elements, +// Filtering, +// Ordering, +// ArrayOps, +// Snapshots, +// Mapping, +// SetOps, +// Aggregates, +// Guards, +// Generators, +// StringJoinOps, +// SequenceOps, +// RangeOps, +// Numeric, +// ComparableOps, +// CommonArrays, +// PlatformSpecialized, +// PlatformSpecializedJS, + { emptySequence() } + ) + require(args.size == 1) { "Expecting Kotlin project home path as an argument" } val baseDir = File(args.first()) - val outDir = baseDir.resolve("libraries/stdlib/src/generated") - require(outDir.exists()) { "$outDir doesn't exist!" } - - val jsCoreDir = baseDir.resolve("js/js.libraries/src/core/generated") - require(jsCoreDir.exists()) { "$jsCoreDir doesn't exist!" } - - generateCollectionsAPI(outDir) - generateCollectionsJsAPI(jsCoreDir) - generateCommonAPI(baseDir.resolve("libraries/stdlib/common/src/generated")) - -} - -val commonGenerators = sequenceOf( - ::elements, - ::filtering, - ::ordering, - ::arrays, - ::snapshots, - ::mapping, - ::sets, - ::aggregates, - ::guards, - ::generators, - ::strings, - ::sequences, - ::ranges, - ::numeric, - ::comparables, - CommonArrays::templates -) - -fun generateCollectionsAPI(outDir: File) { - - val templates = (commonGenerators + ::specialJVM).flatMap { it().sortedBy { it.signature }.asSequence() } - - templates.groupByFileAndWrite(outDir, Platform.JVM) -} - -fun generateCollectionsJsAPI(outDir: File) { - (commonGenerators + ::specialJS).flatMap { it().sortedBy { it.signature }.asSequence() } - .groupByFileAndWrite(outDir, Platform.JS, { "_${it.name.capitalize()}Js.kt"}) -} - -fun generateCommonAPI(outDir: File) { - (commonGenerators + ::specialJVM).flatMap { it().sortedBy { it.signature }.asSequence() } - .groupByFileAndWrite(outDir, platform = Platform.Common) -} - - - - -private fun Sequence.groupByFileAndWrite( - outDir: File, - platform: Platform, - fileNameBuilder: (SourceFile) -> String = { "_${it.name.capitalize()}.kt" } -) { - val groupedConcreteFunctions = map { it.instantiate(platform) }.flatten().groupBy { it.sourceFile } - - for ((sourceFile, functions) in groupedConcreteFunctions) { - val file = outDir.resolve(fileNameBuilder(sourceFile)) - functions.writeTo(file, sourceFile, platform) + fun File.resolveExistingDir(subpath: String) = resolve(subpath).also { + require(it.isDirectory) { "Directory $it doesn't exist"} } -} -private fun List.writeTo(file: File, sourceFile: SourceFile, platform: Platform?) { - println("Generating file: $file") + val commonDir = baseDir.resolveExistingDir("libraries/stdlib/common/src/generated") + val jvmDir = baseDir.resolveExistingDir("libraries/stdlib/src/generated") + val jsDir = baseDir.resolveExistingDir("js/js.libraries/src/core/generated") - FileWriter(file).use { writer -> - if (sourceFile.multifile) { - writer.appendln("@file:kotlin.jvm.JvmMultifileClass") - } - - writer.appendln("@file:kotlin.jvm.JvmName(\"${sourceFile.jvmClassName}\")") - if (platform == Platform.JVM) - writer.appendln("@file:kotlin.jvm.JvmVersion") - writer.appendln() - - writer.append("package ${sourceFile.packageName ?: "kotlin"}\n\n") - writer.append("$COMMON_AUTOGENERATED_WARNING\n\n") - if (platform == Platform.JS) { - writer.appendln("import kotlin.js.*") - if (sourceFile == SourceFile.Arrays) { - writer.appendln("import primitiveArrayConcat") - writer.appendln("import withType") - } - } - writer.appendln("import kotlin.comparisons.*") - - if (platform != Platform.Common && sourceFile == SourceFile.Sequences) { - writer.appendln("import kotlin.coroutines.experimental.*") - } - - writer.appendln() - - for (f in this) { - f.textBuilder(writer) + templateGroups.groupByFileAndWrite { (platform, source) -> + // File("build/out/$platform/$source.kt") + when (platform) { + Platform.Common -> commonDir.resolve("_${source.name.capitalize()}.kt") + Platform.JVM -> jvmDir.resolve("_${source.name.capitalize()}.kt") + Platform.JS -> jsDir.resolve("_${source.name.capitalize()}Js.kt") + Platform.Native -> error("Native is unsupported yet") } } } diff --git a/libraries/tools/kotlin-stdlib-gen/src/templates/dsl/Writers.kt b/libraries/tools/kotlin-stdlib-gen/src/templates/dsl/Writers.kt index c0f3bffb6ed..1eb5d5d41e6 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/templates/dsl/Writers.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/templates/dsl/Writers.kt @@ -16,10 +16,14 @@ package templates -import generators.COMMON_AUTOGENERATED_WARNING import java.io.File import java.io.FileWriter +val COMMON_AUTOGENERATED_WARNING: String = """// +// NOTE THIS FILE IS AUTO-GENERATED by the GenerateStandardLib.kt +// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib +//""" + data class PlatformSourceFile( val platform: Platform, val sourceFile: SourceFile diff --git a/libraries/tools/kotlin-stdlib-gen/src/templates/engine/Engine.kt b/libraries/tools/kotlin-stdlib-gen/src/templates/engine/Engine.kt deleted file mode 100644 index d7c4398966c..00000000000 --- a/libraries/tools/kotlin-stdlib-gen/src/templates/engine/Engine.kt +++ /dev/null @@ -1,462 +0,0 @@ -package templates - -import templates.Family.* -import templates.Family.Collections -import java.io.StringReader -import java.util.* - -open class BaseSpecializedProperty { - protected open fun onKeySet(key: TKey) {} - - var default: TValue? = null - protected set -} - -open class PlatformSpecializedProperty() : BaseSpecializedProperty() { - private val valueBuilders = HashMap TValue)>>() - - operator fun get(platform: Platform, key: TKey): TValue? = run { - valueBuilders[platform]?.get(key) - ?: valueBuilders[null]?.get(key) - ?: valueBuilders[platform]?.get(null) - ?: valueBuilders[null]?.get(null) - }?.let { it(key) } - - fun isSpecializedFor(platform: Platform, key: TKey): Boolean - = valueBuilders[platform]?.contains(key) == true || valueBuilders[null]?.contains(key) == true - - operator fun set(platform: Platform?, keys: Collection, value: TValue) { - if (platform == null && keys.isEmpty()) default = value - set(platform, keys, { value }) - } - operator fun set(platform: Platform?, keys: Collection, value: (TKey)->TValue) { - val valueBuilders = valueBuilders.getOrPut(platform) { hashMapOf() } - if (keys.isEmpty()) - valueBuilders[null] = value; - else - for (key in keys) { - valueBuilders[key] = value - onKeySet(key) - } - } - -} - -open class SpecializedProperty() : BaseSpecializedProperty() { - private val valueBuilders = HashMap TValue)>() - - operator fun get(key: TKey): TValue? = (valueBuilders[key] ?: valueBuilders[null])?.let { it(key) } - - - operator fun set(keys: Collection, value: TValue) { - if (keys.isEmpty()) default = value - set(keys, { value }) - } - operator fun set(keys: Collection, value: (TKey)->TValue) { - if (keys.isEmpty()) - valueBuilders[null] = value; - else - for (key in keys) { - valueBuilders[key] = value - onKeySet(key) - } - } - -} - -operator fun SpecializedProperty.invoke(vararg keys: TKey, valueBuilder: (TKey) -> TValue) = set(keys.asList(), valueBuilder) -operator fun SpecializedProperty.invoke(value: TValue, vararg keys: TKey) = set(keys.asList(), value) -operator fun PlatformSpecializedProperty.invoke(vararg keys: TKey, valueBuilder: (TKey) -> TValue) = set(null, keys.asList(), valueBuilder) -operator fun PlatformSpecializedProperty.invoke(value: TValue, vararg keys: TKey) = set(null, keys.asList(), value) -operator fun PlatformSpecializedProperty.invoke(platform: Platform, vararg keys: TKey, valueBuilder: (TKey) -> TValue) = set(platform, keys.asList(), valueBuilder) -operator fun PlatformSpecializedProperty.invoke(platform: Platform, value: TValue, vararg keys: TKey) = set(platform, keys.asList(), value) - -typealias FamilyProperty = SpecializedProperty -typealias PlatformProperty = SpecializedProperty -typealias PlatformFamilyProperty = PlatformSpecializedProperty -typealias PlatformPrimitiveProperty = PlatformSpecializedProperty - -class DeprecationProperty() : PlatformFamilyProperty() -operator fun DeprecationProperty.invoke(value: String, vararg keys: Family) = set(null, keys.asList(), Deprecation(value)) - -class DocProperty() : PlatformFamilyProperty() -operator fun DocProperty.invoke(vararg keys: Family, valueBuilder: DocExtensions.(Family) -> String) = set(null, keys.asList(), { f -> valueBuilder(DocExtensions, f) }) - -class InlineProperty : PlatformFamilyProperty() -operator fun InlineProperty.invoke(vararg keys: Family) = set(null, keys.asList(), Inline.Yes) -operator fun InlineProperty.invoke(value: Boolean, vararg keys: Family) = set(null, keys.asList(), if (value) Inline.Yes else Inline.No) - -class ConcreteFunction(val textBuilder: (Appendable) -> Unit, val sourceFile: SourceFile) - -class GenericFunction(val signature: String, val keyword: String = "fun") { - - - val defaultFamilies = Family.defaultFamilies - val defaultPrimitives = PrimitiveType.defaultPrimitives - val numericPrimitives = PrimitiveType.numericPrimitives - - var toNullableT: Boolean = false - - val receiverAsterisk = PlatformFamilyProperty() - - val buildFamilies = PlatformProperty>().apply { - invoke(defaultFamilies) - } - //val buildFamilies = LinkedHashSet(defaultFamilies) - val buildFamilyPrimitives = PlatformFamilyProperty>().apply { - invoke(defaultPrimitives) - } - - val customReceiver = PlatformFamilyProperty() - val customSignature = PlatformFamilyProperty() - val deprecate = DeprecationProperty() - val doc = DocProperty() - val platformName = PlatformPrimitiveProperty() - val inline = InlineProperty() - val jvmOnly = FamilyProperty() - val since = PlatformFamilyProperty() - val typeParams = ArrayList() - val returns = PlatformFamilyProperty() - val visibility = FamilyProperty() - val operator = FamilyProperty() - val infix = FamilyProperty() - val external = PlatformFamilyProperty() - val body = object : PlatformFamilyProperty() { - override fun onKeySet(key: Family) = include(key) - } - val customPrimitiveBodies = HashMap, String>() - val annotations = PlatformFamilyProperty() - val sourceFile = FamilyProperty() - val sequenceClassification = mutableListOf() - - fun bodyForTypes(family: Family, vararg primitiveTypes: PrimitiveType, b: (PrimitiveType) -> String) { - include(family) - for (primitive in primitiveTypes) { - customPrimitiveBodies.put(family to primitive, b(primitive)) - } - } - - fun typeParam(t: String) { - typeParams.add(t) - } - - fun sequenceClassification(vararg sequenceClass: SequenceClass) { - sequenceClassification.addAll(sequenceClass) - } - - fun exclude(vararg families: Family) { - buildFamilies(buildFamilies.default!! - families) - } - - fun only(vararg families: Family) { - buildFamilies(families.toSet()) - } - - fun only(platform: Platform, vararg families: Family) { - require(families.isNotEmpty()) { "Need to specify families" } - buildFamilies(families.toSet(), platform) - } - - fun only(vararg primitives: PrimitiveType) { - only(primitives.asList()) - } - - fun only(primitives: Collection) { - buildFamilyPrimitives(primitives.toSet()) - } - - fun onlyPrimitives(family: Family, vararg primitives: PrimitiveType) { - buildFamilyPrimitives(family) { primitives.toSet() } - } - - fun onlyPrimitives(family: Family, primitives: Set) { - buildFamilyPrimitives(family) { primitives } - } - - fun include(vararg families: Family) { - buildFamilies(buildFamilies.default!! + families) - } - - fun exclude(vararg p: PrimitiveType) { - buildFamilyPrimitives(buildFamilyPrimitives.default!! - p) - } - - fun instantiate(platform: Platform, vararg families: Family = Family.values()): List { - return families - .sortedBy { it.ordinal } - .filter { buildFamilies[platform]!!.contains(it) } - .filter { platform == Platform.JVM || jvmOnly[it] != true } - .flatMap { family -> instantiate(family, platform) } - } - - fun instantiate(f: Family, platform: Platform): List { - val onlyPrimitives = buildFamilyPrimitives[platform, f]!! - - if (f.isPrimitiveSpecialization || buildFamilyPrimitives.isSpecializedFor(platform, f)) { - return (onlyPrimitives).sortedBy { it.ordinal } - .map { primitive -> ConcreteFunction( { build(it, f, primitive, platform) }, sourceFileFor(f) ) } - } else { - return listOf(ConcreteFunction( { build(it, f, null, platform) }, sourceFileFor(f) )) - } - } - - private fun sourceFileFor(f: Family) = sourceFile[f] ?: getDefaultSourceFile(f) - - private fun getDefaultSourceFile(f: Family): SourceFile = when (f) { - Iterables, Collections, Lists -> SourceFile.Collections - Sequences -> SourceFile.Sequences - Sets -> SourceFile.Sets - Ranges, RangesOfPrimitives, ProgressionsOfPrimitives -> SourceFile.Ranges - ArraysOfObjects, InvariantArraysOfObjects, ArraysOfPrimitives -> SourceFile.Arrays - Maps -> SourceFile.Maps - Strings -> SourceFile.Strings - CharSequences -> SourceFile.Strings - Primitives, Generic -> SourceFile.Misc - } - -/* - fun build(vararg families: Family = Family.values()): String { - val builder = StringBuilder() - for (family in families.sortedBy { it.name }) { - if (buildFamilies.contains(family)) - build(builder, family) - } - return builder.toString() - } - - fun build(builder: StringBuilder, f: Family) { - val onlyPrimitives = buildFamilyPrimitives[f] - if (f.isPrimitiveSpecialization || onlyPrimitives != null) { - for (primitive in (onlyPrimitives ?: buildPrimitives).sortedBy { it.name }) - build(builder, f, primitive) - } else { - build(builder, f, null) - } - } -*/ - - fun build(builder: Appendable, f: Family, primitive: PrimitiveType?, platform: Platform) { - val headerOnly: Boolean = platform == Platform.Common - val hasOptionalParams = (customSignature[platform, f] ?: signature).contains("=") - val returnType = returns[platform, f] ?: throw RuntimeException("No return type specified for $signature") - - fun renderType(expression: String, receiver: String, self: String): String { - val t = StringTokenizer(expression, " \t\n,:()<>?.", true) - val answer = StringBuilder() - - while (t.hasMoreTokens()) { - val token = t.nextToken() - answer.append(when (token) { - "RECEIVER" -> receiver - "SELF" -> self - "PRIMITIVE" -> primitive?.name ?: token - "SUM" -> { - when (primitive) { - PrimitiveType.Byte, PrimitiveType.Short, PrimitiveType.Char -> "Int" - else -> primitive - } - } - "ZERO" -> when (primitive) { - PrimitiveType.Double -> "0.0" - PrimitiveType.Float -> "0.0f" - PrimitiveType.Long -> "0L" - else -> "0" - } - "ONE" -> when (primitive) { - PrimitiveType.Double -> "1.0" - PrimitiveType.Float -> "1.0f" - PrimitiveType.Long -> "1L" - else -> "1" - } - "-ONE" -> when (primitive) { - PrimitiveType.Double -> "-1.0" - PrimitiveType.Float -> "-1.0f" - PrimitiveType.Long -> "-1L" - else -> "-1" - } - "TCollection" -> { - when (f) { - CharSequences, Strings -> "Appendable" - else -> renderType("MutableCollection", receiver, self) - } - } - "T" -> { - when (f) { - Generic -> "T" - CharSequences, Strings -> "Char" - Maps -> "Map.Entry" - else -> primitive?.name ?: token - } - } - "TRange" -> { - when (f) { - Generic -> "Range" - else -> primitive!!.name + "Range" - } - } - "TProgression" -> { - when (f) { - Generic -> "Progression" - else -> primitive!!.name + "Progression" - } - } - else -> token - }) - } - - return answer.toString() - } - - val isAsteriskOrT = if (receiverAsterisk[platform, f] == true) "*" else "T" - val self = (when (f) { - Iterables -> "Iterable<$isAsteriskOrT>" - Collections -> "Collection<$isAsteriskOrT>" - Lists -> "List<$isAsteriskOrT>" - Maps -> "Map" - Sets -> "Set<$isAsteriskOrT>" - Sequences -> "Sequence<$isAsteriskOrT>" - InvariantArraysOfObjects -> "Array" - ArraysOfObjects -> "Array<${isAsteriskOrT.replace("T", "out T")}>" - Strings -> "String" - CharSequences -> "CharSequence" - Ranges -> "ClosedRange<$isAsteriskOrT>" - ArraysOfPrimitives -> primitive?.let { it.name + "Array" } ?: throw IllegalArgumentException("Primitive array should specify primitive type") - RangesOfPrimitives -> primitive?.let { it.name + "Range" } ?: throw IllegalArgumentException("Primitive range should specify primitive type") - ProgressionsOfPrimitives -> primitive?.let { it.name + "Progression" } ?: throw IllegalArgumentException("Primitive progression should specify primitive type") - Primitives -> primitive?.let { it.name } ?: throw IllegalArgumentException("Primitive should specify primitive type") - Generic -> "T" - }) - - val receiver = (customReceiver[platform, f] ?: self).let { renderType(it, it, self) } - - fun String.renderType(): String = renderType(this, receiver, self) - - fun effectiveTypeParams(): List { - val parameters = typeParams.mapTo(mutableListOf()) { parseTypeParameter(it.renderType()) } - - if (f == Generic) { - if (parameters.none { it.name == "T" }) - parameters.add(TypeParameter("T")) - return parameters - } - else if (primitive == null && f != Strings && f != CharSequences) { - val mentionedTypes = parseTypeRef(receiver).mentionedTypes() + parameters.flatMap { it.mentionedTypeRefs() } - val implicitTypeParameters = mentionedTypes.filter { it.name.all(Char::isUpperCase) } - for (implicit in implicitTypeParameters.reversed()) { - if (implicit.name != "*" && parameters.none { it.name == implicit.name }) { - parameters.add(0, TypeParameter(implicit.name)) - } - } - - return parameters - } else { - // substituted T is no longer a parameter - val renderedT = "T".renderType() - return parameters.filterNot { it.name == renderedT } - } - } - - doc[platform, f]?.let { methodDoc -> - builder.append("/**\n") - StringReader(methodDoc.trim()).forEachLine { line -> - builder.append(" * ").append(line.trim()).append("\n") - } - if (f == Sequences && sequenceClassification.isNotEmpty()) { - builder.append(" *\n") - builder.append(" * The operation is ${sequenceClassification.joinToString(" and ") { "_${it}_" }}.\n") - } - builder.append(" */\n") - } - - deprecate[platform, f]?.let { deprecated -> - val args = listOfNotNull( - "\"${deprecated.message}\"", - deprecated.replaceWith?.let { "ReplaceWith(\"$it\")" }, - deprecated.level.let { if (it != DeprecationLevel.WARNING) "level = DeprecationLevel.$it" else null } - ) - builder.append("@Deprecated(${args.joinToString(", ")})\n") - } - - if (!f.isPrimitiveSpecialization && primitive != null) { - platformName[platform, primitive] - ?.replace("", primitive.name) - ?.let { platformName -> builder.append("@kotlin.jvm.JvmName(\"${platformName}\")\n")} - } - - if (jvmOnly[f] ?: false) { - builder.append("@kotlin.jvm.JvmVersion\n") - } - since[platform, f]?.let { since -> - builder.append("@SinceKotlin(\"$since\")\n") - } - - annotations[platform, f]?.let { builder.append(it.trimIndent()).append('\n') } - - if (inline[platform, f] == Inline.Only) { - builder.append("@kotlin.internal.InlineOnly").append('\n') - } - - builder.append(visibility[f] ?: "public").append(' ') - if (headerOnly && !hasOptionalParams) { - builder.append("expect ") - } - if (external[platform, f] == true) - builder.append("external ") - if (inline[platform, f]?.isInline() == true) - builder.append("inline ") - if (infix[f] == true) - builder.append("infix ") - if (operator[f] == true) - builder.append("operator ") - - builder.append("$keyword ") - - val types = effectiveTypeParams() - if (!types.isEmpty()) { - builder.append(types.joinToString(separator = ", ", prefix = "<", postfix = "> ", transform = { it.original })) - } - - val receiverType = (if (toNullableT) receiver.replace("T>", "T?>") else receiver).renderType() - - builder.append(receiverType) - if (receiverType.isNotEmpty()) builder.append('.') - builder.append("${(customSignature[platform, f] ?: signature).renderType()}: ${returnType.renderType()}") - - if (headerOnly && !hasOptionalParams) { - builder.append("\n\n") - return - } - - if (keyword == "fun") builder.append(" {") - - val body = ( - primitive?.let { customPrimitiveBodies[f to primitive] } ?: - body[platform, f] ?: - deprecate[platform, f]?.replaceWith?.let { "return $it" } ?: - throw RuntimeException("No body specified for $signature for ${f to primitive}") - ).trim('\n') - val indent: Int = body.takeWhile { it == ' ' }.length - - builder.append('\n') - body.lineSequence().forEach { - var count = indent - val line = it.dropWhile { count-- > 0 && it == ' ' }.renderType() - if (!line.isEmpty()) { - builder.append(" ").append(line) - builder.append("\n") - } - } - if (keyword == "fun") builder.append("}\n") - builder.append("\n") - } - -} - -infix fun MutableList.add(item: GenericFunction) = add(item) -infix fun MutableList.addAll(items: Iterable) = this.addAll(elements = items) - -fun f(signature: String, init: GenericFunction.() -> Unit) = GenericFunction(signature).apply(init) - -fun pval(signature: String, init: GenericFunction.() -> Unit) = GenericFunction(signature, "val").apply(init) - -fun pvar(signature: String, init: GenericFunction.() -> Unit) = GenericFunction(signature, "var").apply(init)