Prototype new version of stdlib generator template DSL

This commit is contained in:
Ilya Gorbunov
2017-09-07 16:23:11 +03:00
parent 9e34e20338
commit e68a6651d2
4 changed files with 716 additions and 1 deletions
@@ -4,7 +4,7 @@ import java.io.*
import templates.*
import templates.Family.*
private val COMMON_AUTOGENERATED_WARNING: String = """//
val COMMON_AUTOGENERATED_WARNING: String = """//
// NOTE THIS FILE IS AUTO-GENERATED by the GenerateStandardLib.kt
// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib
//"""
@@ -0,0 +1,346 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package templates
import templates.Family.*
import java.io.StringReader
import java.util.StringTokenizer
private fun getDefaultSourceFile(f: Family): SourceFile = when (f) {
Iterables, Collections, Lists -> SourceFile.Collections
Sequences -> SourceFile.Sequences
Sets -> SourceFile.Sets
Ranges, RangesOfPrimitives, ProgressionsOfPrimitives -> SourceFile.Ranges
ArraysOfObjects, InvariantArraysOfObjects, ArraysOfPrimitives -> SourceFile.Arrays
Maps -> SourceFile.Maps
Strings -> SourceFile.Strings
CharSequences -> SourceFile.Strings
Primitives, Generic -> SourceFile.Misc
}
@TemplateDsl
class MemberBuilder(
val allowedPlatforms: Set<Platform>,
val platform: Platform,
var family: Family,
var sourceFile: SourceFile = getDefaultSourceFile(family),
var primitive: PrimitiveType? = null
) {
lateinit var keyword: Keyword // fun/val/var
lateinit var signature: String // name and params
val f get() = family
var hasPlatformSpecializations: Boolean = false
private set
var doc: String? = null; private set
val sequenceClassification = mutableListOf<SequenceClass>()
var deprecate: Deprecation? = null; private set
var since: String? = null; private set
var platformName: String? = null; private set
var visibility: String? = null; private set
var external: Boolean = false; private set
var inline: Inline = Inline.No; private set
var infix: Boolean = false; private set
var operator: Boolean = false; private set
val typeParams = mutableListOf<String>()
var customReceiver: String? = null; private set
var receiverAsterisk: Boolean = false; private set
var toNullableT: Boolean = false; private set
var returns: String? = null; private set
var body: String? = null; private set
val annotations: MutableList<String> = mutableListOf()
fun sourceFile(file: SourceFile) { sourceFile = file }
fun deprecate(value: Deprecation) { deprecate = value }
fun deprecate(value: String) { deprecate = Deprecation(value)
}
fun platformName(name: String) { platformName = name }
fun visibility(value: String) { visibility = value }
fun external(value: Boolean = true) { external = value }
fun operator(value: Boolean = true) { operator = value }
fun infix(value: Boolean = true) { infix = value }
fun inline(value: Inline = Inline.Yes, suppressWarning: Boolean = false) {
inline = value
if (suppressWarning) {
require(value == Inline.Yes)
annotation("""@Suppress("NOTHING_TO_INLINE")""")
}
}
fun inlineOnly() { inline = Inline.Only
}
fun receiver(value: String) { customReceiver = value }
fun signature(value: String) { signature = value }
fun returns(type: String) { returns = type }
fun typeParam(typeParameterName: String) {
typeParams += typeParameterName
}
fun annotation(annotation: String) {
annotations += annotation
}
fun sequenceClassification(vararg sequenceClass: SequenceClass) {
sequenceClassification += sequenceClass
}
fun doc(valueBuilder: DocExtensions.() -> String) {
doc = valueBuilder(DocExtensions)
}
fun body(valueBuilder: () -> String) {
body = valueBuilder()
}
fun on(platform: Platform, action: () -> Unit) {
require(platform in allowedPlatforms) { "Platform $platform is not in the list of allowed platforms $allowedPlatforms" }
if (this.platform == platform)
action()
else {
hasPlatformSpecializations = true
}
}
fun specialFor(f: Family, action: () -> Unit) {
if (family == f)
action()
}
fun build(builder: Appendable) {
// TODO: legacy mode when all is headerOnly + no_impl
// except functions with optional parameters - they are common + no_impl
val headerOnly: Boolean = platform == Platform.Common && hasPlatformSpecializations
val isImpl: Boolean = platform != Platform.Common && Platform.Common in allowedPlatforms
val returnType = returns ?: throw RuntimeException("No return type specified for $signature")
fun renderType(expression: String, receiver: String, self: String): String {
val t = StringTokenizer(expression, " \t\n,:()<>?.", true)
val answer = StringBuilder()
while (t.hasMoreTokens()) {
val token = t.nextToken()
answer.append(when (token) {
"RECEIVER" -> receiver
"SELF" -> self
"PRIMITIVE" -> primitive?.name ?: token
"SUM" -> {
when (primitive) {
PrimitiveType.Byte, PrimitiveType.Short, PrimitiveType.Char -> "Int"
else -> primitive
}
}
"ZERO" -> when (primitive) {
PrimitiveType.Double -> "0.0"
PrimitiveType.Float -> "0.0f"
PrimitiveType.Long -> "0L"
else -> "0"
}
"ONE" -> when (primitive) {
PrimitiveType.Double -> "1.0"
PrimitiveType.Float -> "1.0f"
PrimitiveType.Long -> "1L"
else -> "1"
}
"-ONE" -> when (primitive) {
PrimitiveType.Double -> "-1.0"
PrimitiveType.Float -> "-1.0f"
PrimitiveType.Long -> "-1L"
else -> "-1"
}
"TCollection" -> {
when (family) {
CharSequences, Strings -> "Appendable"
else -> renderType("MutableCollection<in T>", receiver, self)
}
}
"T" -> {
when (family) {
Generic -> "T"
CharSequences, Strings -> "Char"
Maps -> "Map.Entry<K, V>"
else -> primitive?.name ?: token
}
}
"TRange" -> {
when (family) {
Generic -> "Range<T>"
else -> primitive!!.name + "Range"
}
}
"TProgression" -> {
when (family) {
Generic -> "Progression<out T>"
else -> primitive!!.name + "Progression"
}
}
else -> token
})
}
return answer.toString()
}
val isAsteriskOrT = if (receiverAsterisk) "*" else "T"
val self = (when (family) {
Iterables -> "Iterable<$isAsteriskOrT>"
Collections -> "Collection<$isAsteriskOrT>"
Lists -> "List<$isAsteriskOrT>"
Maps -> "Map<out K, V>"
Sets -> "Set<$isAsteriskOrT>"
Sequences -> "Sequence<$isAsteriskOrT>"
InvariantArraysOfObjects -> "Array<T>"
ArraysOfObjects -> "Array<${isAsteriskOrT.replace("T", "out T")}>"
Strings -> "String"
CharSequences -> "CharSequence"
Ranges -> "ClosedRange<$isAsteriskOrT>"
ArraysOfPrimitives -> primitive?.let { it.name + "Array" } ?: throw IllegalArgumentException("Primitive array should specify primitive type")
RangesOfPrimitives -> primitive?.let { it.name + "Range" } ?: throw IllegalArgumentException("Primitive range should specify primitive type")
ProgressionsOfPrimitives -> primitive?.let { it.name + "Progression" } ?: throw IllegalArgumentException("Primitive progression should specify primitive type")
Primitives -> primitive?.let { it.name } ?: throw IllegalArgumentException("Primitive should specify primitive type")
Generic -> "T"
})
val receiver = (customReceiver ?: self).let { renderType(it, it, self) }
fun String.renderType(): String = renderType(this, receiver, self)
fun effectiveTypeParams(): List<TypeParameter> {
val parameters = typeParams.mapTo(mutableListOf()) { parseTypeParameter(it.renderType()) }
if (family == Generic) {
if (parameters.none { it.name == "T" })
parameters.add(TypeParameter("T"))
return parameters
} else if (primitive == null && family != Strings && family != CharSequences) {
val mentionedTypes = parseTypeRef(receiver).mentionedTypes() + parameters.flatMap { it.mentionedTypeRefs() }
val implicitTypeParameters = mentionedTypes.filter { it.name.all(Char::isUpperCase) }
for (implicit in implicitTypeParameters.reversed()) {
if (implicit.name != "*" && parameters.none { it.name == implicit.name }) {
parameters.add(0, TypeParameter(implicit.name))
}
}
return parameters
} else {
// substituted T is no longer a parameter
val renderedT = "T".renderType()
return parameters.filterNot { it.name == renderedT }
}
}
doc?.let { methodDoc ->
builder.append("/**\n")
StringReader(methodDoc.trim()).forEachLine { line ->
builder.append(" * ").append(line.trim()).append("\n")
}
if (family == Sequences && sequenceClassification.isNotEmpty()) {
builder.append(" *\n")
builder.append(" * The operation is ${sequenceClassification.joinToString(" and ") { "_${it}_" }}.\n")
}
builder.append(" */\n")
}
deprecate?.let { deprecated ->
val args = listOfNotNull(
"\"${deprecated.message}\"",
deprecated.replaceWith?.let { "ReplaceWith(\"$it\")" },
deprecated.level.let { if (it != DeprecationLevel.WARNING) "level = DeprecationLevel.$it" else null }
)
builder.append("@Deprecated(${args.joinToString(", ")})\n")
}
if (!f.isPrimitiveSpecialization && primitive != null) {
platformName
?.replace("<T>", primitive!!.name)
?.let { platformName -> builder.append("@kotlin.jvm.JvmName(\"${platformName}\")\n") }
}
since?.let { since ->
builder.append("@SinceKotlin(\"$since\")\n")
}
annotations.forEach { builder.append(it.trimIndent()).append('\n') }
if (inline == Inline.Only) {
builder.append("@kotlin.internal.InlineOnly").append('\n')
}
listOfNotNull(
visibility ?: "public",
"header".takeIf { headerOnly },
"impl".takeIf { isImpl },
"external".takeIf { external },
"inline".takeIf { inline.isInline() },
"infix".takeIf { infix },
"operator".takeIf { operator },
keyword.value
).forEach { builder.append(it).append(' ') }
val types = effectiveTypeParams()
if (!types.isEmpty()) {
builder.append(types.joinToString(separator = ", ", prefix = "<", postfix = "> ", transform = { it.original }))
}
val receiverType = (if (toNullableT) receiver.replace("T>", "T?>") else receiver).renderType()
builder.append(receiverType)
if (receiverType.isNotEmpty()) builder.append('.')
builder.append("${signature.renderType()}: ${returnType.renderType()}")
if (headerOnly) {
builder.append("\n\n")
return
}
if (keyword == Keyword.Function) builder.append(" {")
val body = (body ?:
deprecate?.replaceWith?.let { "return $it" } ?:
throw RuntimeException("No body specified for $signature for ${family to primitive}")
).trim('\n')
val indent: Int = body.takeWhile { it == ' ' }.length
builder.append('\n')
body.lineSequence().forEach {
var count = indent
val line = it.dropWhile { count-- > 0 && it == ' ' }.renderType()
if (!line.isEmpty()) {
builder.append(" ").append(line)
builder.append("\n")
}
}
if (keyword == Keyword.Function) builder.append("}\n")
builder.append("\n")
}
}
@@ -0,0 +1,293 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package templates
import kotlin.coroutines.experimental.buildSequence
@DslMarker
annotation class TemplateDsl
enum class Keyword(val value: String) {
Function("fun"),
Value("val"),
Variable("var");
}
typealias MemberBuildAction = MemberBuilder.() -> Unit
typealias MemberBuildActionP<TParam> = MemberBuilder.(TParam) -> Unit
private fun def(signature: String, memberKind: Keyword): MemberBuildAction = {
this.signature = signature
this.keyword = memberKind
}
fun fn(defaultSignature: String): MemberBuildAction = def(defaultSignature, Keyword.Function)
fun fn(defaultSignature: String, setup: FamilyPrimitiveMemberDsl.() -> Unit): MemberTemplate =
FamilyPrimitiveMemberDsl().apply {
builder(fn(defaultSignature))
setup()
}
fun MemberBuildAction.byTwoPrimitives(setup: PairPrimitiveMemberDsl.() -> Unit): MemberTemplate =
PairPrimitiveMemberDsl().apply {
builder(this@byTwoPrimitives)
setup()
}
fun pval(name: String, setup: FamilyPrimitiveMemberDsl.() -> Unit): MemberTemplate =
FamilyPrimitiveMemberDsl().apply {
builder(def(name, Keyword.Value))
setup()
}
fun pvar(name: String, setup: FamilyPrimitiveMemberDsl.() -> Unit): MemberTemplate =
FamilyPrimitiveMemberDsl().apply {
builder(def(name, Keyword.Variable))
setup()
}
interface MemberTemplate {
/** Specifies which platforms this member template should be generated for */
fun platforms(vararg platforms: Platform)
fun instantiate(): Sequence<MemberBuilder>
/** Registers parameterless member builder function */
fun builder(b: MemberBuildAction)
}
abstract class GenericMemberDsl<TParam> : MemberTemplate {
sealed class BuildAction {
class Generic(val action: MemberBuildAction) : BuildAction() {
operator fun invoke(builder: MemberBuilder) { action(builder) }
}
class Parametrized(val action: MemberBuildActionP<*>) : BuildAction() {
@Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER", "UNCHECKED_CAST")
operator fun <TParam> invoke(builder: MemberBuilder, p: @kotlin.internal.NoInfer TParam) {
(action as MemberBuildActionP<TParam>).invoke(builder, p)
}
}
}
val buildActions = mutableListOf<BuildAction>()
private var targetPlatforms = setOf(*Platform.values())
override fun platforms(vararg platforms: Platform) {
targetPlatforms = setOf(*platforms)
}
private var filterPredicate: ((Family, TParam) -> Boolean)? = null
/** Sets the filter predicate that is applied to a produced sequence of variations. */
fun filter(predicate: (Family, TParam) -> Boolean) {
this.filterPredicate = predicate
}
override fun builder(b: MemberBuildAction) { buildActions += BuildAction.Generic(b) }
/** Registers member builder function with the parameter(s) of this DSL */
fun builderWith(b: MemberBuildActionP<TParam>) { buildActions += BuildAction.Parametrized(b) }
/** Provides the sequence of member variation parameters */
protected abstract fun parametrize(): Sequence<Pair<Family, TParam>>
private fun Sequence<Pair<Family, TParam>>.applyFilter() =
filterPredicate?.let { predicate ->
filter { (family, p) -> predicate(family, p) }
} ?: this
override fun instantiate(): Sequence<MemberBuilder> {
val specificPlatforms by lazy { targetPlatforms - Platform.Common }
fun platformMemberBuilders(family: Family, p: TParam) =
if (Platform.Common in targetPlatforms) {
val commonMemberBuilder = createMemberBuilder(Platform.Common, family, p)
mutableListOf(commonMemberBuilder).also { builders ->
if (commonMemberBuilder.hasPlatformSpecializations) {
specificPlatforms.mapTo(builders) {
createMemberBuilder(it, family, p)
}
}
}
} else {
targetPlatforms.map { createMemberBuilder(it, family, p) }
}
return parametrize()
.applyFilter()
.map { (family, p) -> platformMemberBuilders(family, p) }
.flatten()
}
private fun createMemberBuilder(platform: Platform, family: Family, p: TParam): MemberBuilder {
return MemberBuilder(targetPlatforms, platform, family).also { builder ->
for (action in buildActions) {
when (action) {
is BuildAction.Generic -> action(builder)
is BuildAction.Parametrized -> action<TParam>(builder, p)
}
}
}
}
}
private fun defaultPrimitives(f: Family): Set<PrimitiveType> =
if (f.isPrimitiveSpecialization) PrimitiveType.defaultPrimitives else emptySet()
@TemplateDsl
class FamilyPrimitiveMemberDsl : GenericMemberDsl<PrimitiveType?>() {
private val familyPrimitives = mutableMapOf<Family, Set<PrimitiveType>>()
fun include(vararg fs: Family) {
for (f in fs) familyPrimitives[f] = defaultPrimitives(f)
}
fun include(fs: Collection<Family>) {
for (f in fs) familyPrimitives[f] = defaultPrimitives(f)
}
fun includeDefault() {
include(Family.defaultFamilies)
}
fun include(f: Family, primitives: Set<PrimitiveType>) {
familyPrimitives[f] = primitives
}
fun exclude(vararg ps: PrimitiveType) {
val toExclude = ps.toSet()
for (e in familyPrimitives) {
e.setValue(e.value - toExclude)
}
}
override fun parametrize(): Sequence<Pair<Family, PrimitiveType?>> = buildSequence {
for ((family, primitives) in familyPrimitives) {
if (primitives.isEmpty())
yield(family to null)
else
yieldAll(primitives.map { family to it })
}
}
init {
builderWith { p -> primitive = p }
}
}
@TemplateDsl
class PairPrimitiveMemberDsl : GenericMemberDsl<Pair<PrimitiveType, PrimitiveType>>() {
private val familyPrimitives = mutableMapOf<Family, Set<Pair<PrimitiveType, PrimitiveType>>>()
fun include(f: Family, primitives: Collection<Pair<PrimitiveType, PrimitiveType>>) {
familyPrimitives[f] = primitives.toSet()
}
override fun parametrize(): Sequence<Pair<Family, Pair<PrimitiveType, PrimitiveType>>> {
return familyPrimitives
.flatMap { e -> e.value.map { e.key to it } }
.asSequence()
}
init {
builderWith { (p1, p2) -> primitive = p1 }
}
}
/*
val t_copyOfResized = MemberTemplatePar<DefaultParametrization>().apply {
parametrization = buildSequence<DefaultParametrization> {
val allPlatforms = setOf(*Platform.values())
yield(DefaultParametrization(InvariantArraysOfObjects, platforms = allPlatforms))
yieldAll(PrimitiveType.defaultPrimitives.map { DefaultParametrization(ArraysOfPrimitives, it, platforms = allPlatforms) })
}
builder = { p, platform, builder ->
builder.family = if (p.family == InvariantArraysOfObjects && platform == Platform.JS)
ArraysOfObjects else p.family
if (platform == Platform.JVM)
builder.inline = Inline.Only
builder.doc = "Returns new array which is a copy of the original array."
builder.returns = "SELF"
if (platform == Platform.JS && p.family == ArraysOfObjects)
builder.returns = "Array<T>"
if (platform == Platform.JVM) {
builder.body = "return java.util.Arrays.copyOf(this, size)"
} else if (platform == Platform.JS) {
when (p.primitive) {
null ->
builder.body = "return this.asDynamic().slice()"
PrimitiveType.Char, PrimitiveType.Boolean, PrimitiveType.Long ->
builder.body = "return withType(\"${p.primitive}Array\", this.asDynamic().slice())"
else -> {
builder.annotations += """@Suppress("NOTHING_TO_INLINE")"""
builder.inline = Inline.Yes
builder.body = "return this.asDynamic().slice()"
}
}
}
}
}
interface MemberTemplate {
fun instantiate(): Sequence<MemberInstance>
}
class MemberTemplatePar<TParametrization : Parametrization> : MemberTemplate {
val keyword: String = "fun"
lateinit var parametrization: Sequence<TParametrization>
lateinit var builder: (TParametrization, Platform, MemberBuilder) -> Unit
override fun instantiate(): Sequence<MemberInstance> =
parametrization.flatMap {
it.platforms.asSequence().map { p ->
val memberBuilder = MemberBuilder().apply {
builder(it, p, this)
}
MemberInstance(memberBuilder::build, PlatformSourceFile(p, memberBuilder.sourceFile))
}
}
}
*/
/*
class MemberInstance(
val textBuilder: (Appendable) -> Unit,
val platformSourceFile: PlatformSourceFile)
*/
@@ -0,0 +1,76 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package templates
import generators.COMMON_AUTOGENERATED_WARNING
import java.io.File
import java.io.FileWriter
data class PlatformSourceFile(
val platform: Platform,
val sourceFile: SourceFile
)
fun Sequence<MemberTemplate>.groupByFileAndWrite(
fileNameBuilder: (PlatformSourceFile) -> File
) {
val groupedMembers = map { it.instantiate() }.flatten().groupBy {
PlatformSourceFile(it.platform, it.sourceFile)
}
for ((psf, members) in groupedMembers) {
val file = fileNameBuilder(psf)
members.writeTo(file, psf)
}
}
fun List<MemberBuilder>.writeTo(file: File, platformSource: PlatformSourceFile) {
val (platform, sourceFile) = platformSource
println("Generating file: $file")
file.parentFile.mkdirs()
FileWriter(file).use { writer ->
if (sourceFile.multifile) {
writer.appendln("@file:kotlin.jvm.JvmMultifileClass")
}
writer.appendln("@file:kotlin.jvm.JvmName(\"${sourceFile.jvmClassName}\")")
if (platform == Platform.JVM)
writer.appendln("@file:kotlin.jvm.JvmVersion")
writer.appendln()
writer.append("package ${sourceFile.packageName ?: "kotlin"}\n\n")
writer.append("${COMMON_AUTOGENERATED_WARNING}\n\n")
if (platform == Platform.JS) {
writer.appendln("import kotlin.js.*")
if (sourceFile == SourceFile.Arrays) {
writer.appendln("import primitiveArrayConcat")
writer.appendln("import withType")
}
}
writer.appendln("import kotlin.comparisons.*")
if (platform != Platform.Common && sourceFile == SourceFile.Sequences) {
writer.appendln("import kotlin.coroutines.experimental.*")
}
writer.appendln()
for (f in this) {
f.build(writer)
}
}
}