[Commonizer] Add extendable uniform multi-task CLI

native/commonizer/build.gradle.kts

@@ -36,7 +36,7 @@ dependencies {
 
 val runCommonizer by tasks.registering(NoDebugJavaExec::class) {
     classpath(sourceSets.main.get().runtimeClasspath)
-    main = "org.jetbrains.kotlin.descriptors.commonizer.cli.NativeDistributionCommonizerKt"
+    main = "org.jetbrains.kotlin.descriptors.commonizer.cli.CommonizerCLI"
 }
 
 sourceSets {

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/BooleanOptionType.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.descriptors.commonizer.cli
+
+internal class BooleanOptionType(
+    alias: String,
+    description: String,
+    mandatory: Boolean
+) : OptionType<Boolean>(alias, description, mandatory) {
+    private val trueTokens = setOf("1", "on", "yes", "true")
+    private val falseTokens = setOf("0", "off", "no", "false")
+
+    override fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option<Boolean> {
+        val value = rawValue.toLowerCase().let {
+            when (it) {
+                in trueTokens -> true
+                in falseTokens -> false
+                else -> onError("Invalid boolean value: $it")
+            }
+        }
+
+        return Option(this, value)
+    }
+}

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/CliLoggerAdapter.kt

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.descriptors.commonizer.cli
+
+import org.jetbrains.kotlin.util.Logger
+import kotlin.system.exitProcess
+
+internal class CliLoggerAdapter(indentSize: Int) : Logger {
+    private val indent = " ".repeat(indentSize)
+
+    override fun log(message: String) = printlnIndented(message)
+    override fun warning(message: String) = printlnIndented("Warning: $message")
+
+    override fun error(message: String) = fatal(message)
+    override fun fatal(message: String): Nothing {
+        printlnIndented("Error: $message\n")
+        exitProcess(1)
+    }
+
+    private fun printlnIndented(text: String) =
+        if (indent.isEmpty()) println(text)
+        else text.split('\n').forEach { println(indent + it) }
+}

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/NativeDistributionOptionType.kt

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.descriptors.commonizer.cli
+
+import java.io.File
+
+internal object NativeDistributionOptionType : OptionType<File>("distribution-path", "Path to the Kotlin/Native distribution") {
+    override fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option<File> {
+        val file = File(rawValue)
+
+        try {
+            if (!file.isDirectory) onError("Kotlin/Native distribution directory does not exist: $rawValue")
+        } catch (_: Exception) {
+            onError("Access failure to the Kotlin/Native distribution directory: $rawValue")
+        }
+
+        return Option(this, file)
+    }
+}

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/NativeTargetsOptionType.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.descriptors.commonizer.cli
+
+import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.konan.target.KonanTarget
+
+internal object NativeTargetsOptionType : OptionType<List<KonanTarget>>("targets", "Comma-separated list of hardware targets") {
+    private val hostManager = HostManager()
+
+    override fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option<List<KonanTarget>> {
+        val targetNames = rawValue.split(',')
+        if (targetNames.isEmpty()) onError("No hardware targets specified: $rawValue")
+
+        val targets = targetNames.mapTo(HashSet<KonanTarget>()) { targetName ->
+            hostManager.targets[targetName] ?: onError("Unknown hardware target: $targetName")
+        }.toList()
+
+        return Option(this, targets)
+    }
+}

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/Option.kt

@@ -0,0 +1,8 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.descriptors.commonizer.cli
+
+internal class Option<T>(val type: OptionType<T>, val value: T)

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/OptionType.kt

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.descriptors.commonizer.cli
+
+internal abstract class OptionType<T>(
+    val alias: String,
+    val description: String,
+    val mandatory: Boolean = true
+) {
+    abstract fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option<T>
+}

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/OutputOptionType.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.descriptors.commonizer.cli
+
+import java.io.File
+
+internal object OutputOptionType : OptionType<File>("output-path", "Destination for commonized libraries") {
+    override fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option<File> {
+        val file = File(rawValue)
+
+        try {
+            val valid = when {
+                file.isDirectory -> file.listFiles()?.isEmpty() != false
+                file.exists() -> false
+                else -> {
+                    if (!file.mkdirs()) onError("Destination can't be created: $rawValue")
+                    true
+                }
+            }
+
+            if (!valid) onError("Destination is not empty: $rawValue")
+        } catch (_: Exception) {
+            onError("Access failure to the destination directory: $rawValue")
+        }
+
+        return Option(this, file)
+    }
+}

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/Task.kt

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.descriptors.commonizer.cli
+
+import java.util.concurrent.atomic.AtomicInteger
+
+internal abstract class Task(private val options: Collection<Option<*>>) : Comparable<Task> {
+    internal enum class Category(
+        open val prologue: String? = null,
+        open val epilogue: String? = null,
+        open val logEachStep: Boolean = false
+    ) {
+        // Important: the order of entries affects that order of tasks execution
+        INFORMATIONAL,
+        COMMONIZATION(
+            prologue = "Kotlin KLIB commonizer: Please wait while processing libraries.",
+            epilogue = "Kotlin KLIB commonizer: Done.\n",
+            logEachStep = true
+        )
+    }
+
+    abstract val category: Category
+    private val submissionOrder = SUBMISSION_ORDER_GENERATOR.getAndIncrement()
+
+    abstract fun execute(logPrefix: String = "")
+
+    protected inline fun <reified T, reified O : OptionType<T>> getMandatory(nameFilter: (String) -> Boolean = { true }): T {
+        val option = options.filter { it.type is O }.single { nameFilter(it.type.alias) }
+        check(option.type.mandatory)
+
+        @Suppress("UNCHECKED_CAST")
+        return option.value as T
+    }
+
+    protected inline fun <reified T, reified O : OptionType<T>> getOptional(nameFilter: (String) -> Boolean = { true }): T? {
+        val option = options.filter { it.type is O }.singleOrNull { nameFilter(it.type.alias) }
+        if (option != null) check(!option.type.mandatory)
+
+        @Suppress("UNCHECKED_CAST")
+        return option?.value as T?
+    }
+
+    override fun compareTo(other: Task): Int {
+        category.compareTo(other.category).let {
+            if (it != 0) return it
+        }
+
+        this::class.java.name.compareTo(other::class.java.name).let {
+            if (it != 0) return it
+        }
+
+        return submissionOrder - other.submissionOrder
+    }
+
+    companion object {
+        private val SUBMISSION_ORDER_GENERATOR = AtomicInteger(0)
+    }
+}

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/TaskType.kt

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.descriptors.commonizer.cli
+
+internal enum class TaskType(
+    val alias: String,
+    val description: String,
+    val optionTypes: List<OptionType<*>>,
+    val taskConstructor: (Collection<Option<*>>) -> Task
+) {
+    NATIVE_DIST_COMMONIZE(
+        "native-dist-commonize",
+        "Commonize platform-specific libraries in Kotlin/Native distribution",
+        listOf(
+            NativeDistributionOptionType,
+            OutputOptionType,
+            NativeTargetsOptionType,
+            BooleanOptionType(
+                "copy-stdlib",
+                "Boolean (default false);\nwhether to copy Kotlin/Native endorsed libraries to the destination",
+                mandatory = false
+            ),
+            BooleanOptionType(
+                "copy-endorsed-libs",
+                "Boolean (default false);\nwhether to copy Kotlin/Native endorsed libraries to the destination",
+                mandatory = false
+            ),
+            BooleanOptionType("log-stats", "Boolean (default false); Log commonization stats", mandatory = false)
+        ),
+        ::NativeDistributionCommonize
+    ),
+
+    NATIVE_DIST_LIST_TARGETS(
+        "native-dist-print-targets",
+        "Print all hardware targets inside of the Kotlin/Native distribution",
+        listOf(
+            NativeDistributionOptionType
+        ),
+        ::NativeDistributionListTargets
+    );
+
+    companion object {
+        fun getByAlias(alias: String) = values().firstOrNull { it.alias == alias }
+    }
+}

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/cli.kt

@@ -0,0 +1,123 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+@file:JvmName("CommonizerCLI")
+
+package org.jetbrains.kotlin.descriptors.commonizer.cli
+
+import org.jetbrains.kotlin.descriptors.commonizer.cli.Task.Category
+import kotlin.system.exitProcess
+
+fun main(args: Array<String>) {
+    if (args.isEmpty()) printUsageAndExit()
+
+    val tokens = args.iterator()
+    val tasks = mutableListOf<Task>()
+
+    var taskAlias: String? = tokens.next()
+    while (taskAlias != null) {
+        taskAlias = parseTask(taskAlias, tokens, tasks)
+    }
+
+    // execute tasks in a specific order:
+    // - first, execute all informational tasks
+    // - then, all commonization tasks
+    Category.values().forEach { category ->
+        val sortedTasks = tasks.filter { it.category == category }.sorted()
+        if (sortedTasks.isNotEmpty()) {
+            category.prologue?.let(::println)
+
+            sortedTasks.forEachIndexed { index, task ->
+                val logPrefix = if (category.logEachStep) "[Step ${index + 1} of ${sortedTasks.size}] " else ""
+                task.execute(logPrefix)
+            }
+
+            category.epilogue?.let(::println)
+        }
+    }
+}
+
+private fun parseTask(
+    taskAlias: String,
+    tokens: Iterator<String>,
+    tasks: MutableList<Task>
+): String? {
+    val taskType = TaskType.getByAlias(taskAlias) ?: printUsageAndExit("Unknown task $taskAlias")
+    val optionTypes = taskType.optionTypes.associateBy { it.alias }
+    val options = mutableMapOf<String, Option<*>>()
+
+    fun buildOngoingTask() {
+        // check options completeness
+        val missedMandatoryOptions = optionTypes.filterKeys { it !in options }.filterValues { it.mandatory }.keys
+        if (missedMandatoryOptions.isNotEmpty())
+            printUsageAndExit("Mandatory options not specified in task $taskAlias: " + missedMandatoryOptions.joinToString { "-$it" })
+
+        tasks += taskType.taskConstructor(options.values)
+    }
+
+    while (tokens.hasNext()) {
+        val optionAlias = tokens.next().let { token ->
+            if (!token.startsWith('-')) {
+                buildOngoingTask()
+
+                // proceed to the next task
+                return token
+            }
+
+            token.trimStart('-')
+        }
+
+        if (optionAlias in options) printUsageAndExit("Duplicated value for option -$optionAlias in task $taskAlias")
+
+        val optionType = optionTypes[optionAlias] ?: printUsageAndExit("Unknown option -$optionAlias in task $taskAlias")
+
+        val rawValue = if (tokens.hasNext()) tokens.next() else printUsageAndExit("No value for option -$optionAlias in task $taskAlias")
+        val option = optionType.parse(rawValue) { reason ->
+            printUsageAndExit("Failed to parse option -$optionAlias in task $taskAlias: $reason")
+        }
+
+        options[optionAlias] = option
+    }
+
+    buildOngoingTask()
+    return null // no next task
+}
+
+private fun printUsageAndExit(errorMessage: String? = null): Nothing {
+    if (errorMessage != null) {
+        println("Error: $errorMessage")
+        println()
+    }
+
+    fun formatLeft(indent: Int, left: String) = StringBuilder().apply {
+        repeat(indent) { append("    ") }
+        append(left)
+    }
+
+    fun StringBuilder.formatRight(right: String): String {
+        val middleSpace = kotlin.math.max(38 - length, 1)
+        repeat(middleSpace) { append(" ") }
+        append(right)
+        return this.toString()
+    }
+
+    fun formatBoth(indent: Int, left: String, right: String) = formatLeft(indent, left).formatRight(right)
+
+    println("Usage: ${::printUsageAndExit.javaClass.`package`.name}.CommonizerCLI <task> <options> [<task> <options>...]")
+    println()
+    println("Tasks:")
+    for (taskType in TaskType.values()) {
+        println(formatBoth(1, taskType.alias, taskType.description))
+        println(formatLeft(1, if (taskType.optionTypes.isNotEmpty()) "Options:" else "No options."))
+        for (optionType in taskType.optionTypes) {
+            val lines = optionType.description.split('\n')
+            println(formatBoth(2, "-${optionType.alias}", lines.first()))
+            lines.drop(1).forEach { println(StringBuilder().formatRight(it)) }
+        }
+        println()
+    }
+
+    exitProcess(if (errorMessage != null) 1 else 0)
+}

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/cliUtils.kt

@@ -1,35 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.descriptors.commonizer.cli
-
-import org.jetbrains.kotlin.util.Logger
-import kotlin.system.exitProcess
-
-internal fun parseArgs(
-    args: Array<String>,
-    printUsageAndExit: (String) -> Nothing
-): Map<String, List<String>> {
-    val commandLine = mutableMapOf<String, MutableList<String>>()
-    for (index in args.indices step 2) {
-        val key = args[index]
-        if (key[0] != '-') printUsageAndExit("Expected a flag with initial dash: $key")
-        if (index + 1 == args.size) printUsageAndExit("Expected a value after $key")
-        val value = args[index + 1]
-        commandLine.computeIfAbsent(key) { mutableListOf() }.add(value)
-    }
-    return commandLine
-}
-
-internal object CliLoggerAdapter : Logger {
-    override fun log(message: String) = println(message)
-    override fun warning(message: String) = println("Warning: $message")
-
-    override fun error(message: String) = fatal(message)
-    override fun fatal(message: String): Nothing {
-        println("Error: $message\n")
-        exitProcess(1)
-    }
-}

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeDistributionCommonizer.kt

@@ -1,56 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.descriptors.commonizer.cli
-
-import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeDistributionCommonizer
-import org.jetbrains.kotlin.konan.target.HostManager
-import java.io.File
-import kotlin.system.exitProcess
-
-fun main(args: Array<String>) {
-    if (args.isEmpty()) printUsageAndExit()
-
-    val parsedArgs = parseArgs(args, ::printUsageAndExit)
-
-    val repository = parsedArgs["-repository"]?.firstOrNull()?.let(::File) ?: printUsageAndExit("repository not specified")
-
-    val targets = with(HostManager()) {
-        val targetNames = parsedArgs["-target"]?.toSet() ?: printUsageAndExit("no targets specified")
-        targetNames.map { targetName ->
-            targets[targetName] ?: printUsageAndExit("unknown target name: $targetName")
-        }
-    }
-
-    val destination = parsedArgs["-output"]?.firstOrNull()?.let(::File) ?: printUsageAndExit("output not specified")
-
-    val withStats = parsedArgs["-stats"]?.firstOrNull()?.toLowerCase() in setOf("1", "on", "yes", "true")
-
-    NativeDistributionCommonizer(
-        repository = repository,
-        targets = targets,
-        destination = destination,
-        copyStdlib = true,
-        copyEndorsedLibs = true,
-        withStats = withStats,
-        logger = CliLoggerAdapter
-    ).run()
-}
-
-private fun printUsageAndExit(errorMessage: String? = null): Nothing {
-    if (errorMessage != null) {
-        println("Error: $errorMessage")
-        println()
-    }
-
-    println("Usage: commonizer <options>")
-    println("where possible options include:")
-    println("\t-repository <path>\tWork with the specified Kotlin/Native repository")
-    println("\t-target <name>\t\tAdd hardware target to commonization")
-    println("\t-output <path>\t\tDestination of commonized KLIBs")
-    println()
-
-    exitProcess(if (errorMessage != null) 1 else 0)
-}

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.descriptors.commonizer.cli
+
+import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeDistributionCommonizer
+import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_KLIB_DIR
+import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR
+import org.jetbrains.kotlin.konan.target.KonanTarget
+import java.io.File
+
+internal class NativeDistributionListTargets(options: Collection<Option<*>>) : Task(options) {
+    override val category get() = Category.INFORMATIONAL
+
+    override fun execute(logPrefix: String) {
+        val distributionPath = getMandatory<File, NativeDistributionOptionType>()
+
+        val targets = distributionPath.resolve(KONAN_DISTRIBUTION_KLIB_DIR)
+            .resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR)
+            .list()
+            ?.sorted()
+            ?: emptyList()
+
+        println()
+        if (targets.isEmpty())
+            println("No hardware targets found inside of the Kotlin/Native distribution \"$distributionPath\".")
+        else {
+            println("${targets.size} hardware targets found inside of the Kotlin/Native distribution \"$distributionPath\":")
+            targets.forEach(::println)
+        }
+        println()
+    }
+}
+
+internal class NativeDistributionCommonize(options: Collection<Option<*>>) : Task(options) {
+    override val category get() = Category.COMMONIZATION
+
+    override fun execute(logPrefix: String) {
+        val distribution = getMandatory<File, NativeDistributionOptionType>()
+        val destination = getMandatory<File, OutputOptionType>()
+        val targets = getMandatory<List<KonanTarget>, NativeTargetsOptionType>()
+
+        val copyStdlib = getOptional<Boolean, BooleanOptionType> { it == "copy-stdlib" } ?: false
+        val copyEndorsedLibs = getOptional<Boolean, BooleanOptionType> { it == "copy-endorsed-libs" } ?: false
+        val withStats = getOptional<Boolean, BooleanOptionType> { it == "log-stats" } ?: false
+
+        val descriptionSuffix = estimateLibrariesCount(distribution, targets)?.let { " ($it items)" } ?: ""
+        val description = "${logPrefix}Preparing commonized Kotlin/Native libraries for targets $targets$descriptionSuffix"
+
+        println(description)
+
+        NativeDistributionCommonizer(
+            repository = distribution,
+            targets = targets,
+            destination = destination,
+            copyStdlib = copyStdlib,
+            copyEndorsedLibs = copyEndorsedLibs,
+            withStats = withStats,
+            logger = CliLoggerAdapter(2)
+        ).run()
+
+        println("$description: Done")
+    }
+
+    companion object {
+        private fun estimateLibrariesCount(distribution: File, targets: List<KonanTarget>): Int? {
+            val targetNames = targets.map { it.name }
+            return distribution.resolve(KONAN_DISTRIBUTION_KLIB_DIR)
+                .resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR)
+                .listFiles()
+                ?.filter { it.name in targetNames }
+                ?.mapNotNull { it.listFiles() }
+                ?.flatMap { it.toList() }
+                ?.size
+        }
+    }
+}

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt

@@ -47,20 +47,18 @@ class NativeDistributionCommonizer(
         with(ResettableClockMark()) {
             // 1. load modules
             val modulesByTargets = loadModules()
-            logger.log("Loaded lazy (uninitialized) libraries in ${elapsedSinceLast()}")
+            logger.log("* Loaded lazy (uninitialized) libraries in ${elapsedSinceLast()}")
 
             // 2. run commonization
             val result = commonize(modulesByTargets)
-            logger.log("Commonization performed in ${elapsedSinceLast()}")
+            logger.log("* Commonization performed in ${elapsedSinceLast()}")
 
             // 3. write new libraries
             saveModules(modulesByTargets, result)
-            logger.log("Written libraries in ${elapsedSinceLast()}")
+            logger.log("* Written libraries in ${elapsedSinceLast()}")
 
             logger.log("TOTAL: ${elapsedSinceStart()}")
         }
-
-        logger.log("Done.\n")
     }
 
     private fun checkPreconditions() {