diff --git a/samples/settings.gradle b/samples/settings.gradle index ad365fb16bc..cdb2f7b5ef6 100644 --- a/samples/settings.gradle +++ b/samples/settings.gradle @@ -10,6 +10,7 @@ include ':opengl' include ':socket' include ':tetris' include ':tensorflow' +include ':torch' // Android native activity build requires Android SDK. // So temporary switching off for now, as it breaks the build // of other samples if SDK is not present. diff --git a/samples/torch/README.md b/samples/torch/README.md new file mode 100644 index 00000000000..7e465f69502 --- /dev/null +++ b/samples/torch/README.md @@ -0,0 +1,47 @@ +# Torch demo + +Trains a handwritten digit classifier using the [Torch](http://torch.ch) C backend. +Like other Torch clients, most prominently [PyTorch](http://pytorch.org), +this example is built on top of the +[ATen C API](https://github.com/pytorch/pytorch/tree/master/aten), +showing how a Torch client for Kotlin/Native could look like. + +## Installation + +To build [ATen (Torch for C)](https://github.com/pytorch/pytorch/tree/master/aten), +make sure you have Python 2.X and pyyaml installed: + + # macOS: if you don't have pip + sudo easy_install pip + # Linux: if you don't have pip + apt-get -y install python-pip + + # if you don't have pyyaml + sudo pip install pyyaml + +Now + + ./downloadTorch.sh + +will install it into `$HOME/.konan/third-party/torch` (if not yet done). + +To build use `../gradlew build` or `./build.sh`. + + ./downloadMNIST.sh + +will download and unzip the [MNIST dataset](https://en.wikipedia.org/wiki/MNIST_database) of +[70000 labeled handwritten digits](http://yann.lecun.com/exdb/mnist/) for training and testing a classifier. + +Then run + + ../gradlew run + +Alternatively you can run the artifact directly through + + ./build/konan/bin/macbook/HelloTorch.kexe + +You may need to specify `LD_LIBRARY_PATH` or `DYLD_LIBRARY_PATH` to `$HOME/.konan/third-party/torch/lib` +if the ATen dynamic library cannot be found. + +Even on a CPU, training should only take some minutes, +and you should observe a classification accuracy of about 95% on the test dataset. \ No newline at end of file diff --git a/samples/torch/build.gradle b/samples/torch/build.gradle new file mode 100644 index 00000000000..1c5b980bb6d --- /dev/null +++ b/samples/torch/build.gradle @@ -0,0 +1,34 @@ +apply plugin: 'konan' + +konan.targets = ['macbook', 'linux'] + +def torchHome = "${System.getProperty("user.home")}/.konan/third-party/torch" + +task downloadTorch(type: Exec) { + workingDir getProjectDir() + commandLine './downloadTorch.sh' +} + +konanArtifacts { + interop('TorchInterop') { + defFile "src/main/c_interop/torch.def" + includeDirs "${torchHome}/include", "${torchHome}/include/TH" + dependsOn 'downloadTorch' + } + + program('Torch') { + libraries { + artifact 'TorchInterop' + } + linkerOpts "-L${torchHome}/lib -lATen" + } +} + +run.dependsOn 'warning' + +task warning { + doLast { + println "Note: You may need to specify LD_LIBRARY_PATH or DYLD_LIBRARY_PATH env variables to $torchHome/lib if the ATen dynamic library cannot be found." + + } +} \ No newline at end of file diff --git a/samples/torch/build.sh b/samples/torch/build.sh new file mode 100755 index 00000000000..917436eeb4d --- /dev/null +++ b/samples/torch/build.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}" )" && pwd ) + +source "$DIR/../konan.sh" + +$DIR/downloadTorch.sh + +TH_TARGET_DIRECTORY="$HOME/.konan/third-party/torch" + +if [ x$TARGET == x ]; then +case "$OSTYPE" in + darwin*) TARGET=macbook; TF_TARGET=darwin ;; + linux*) TARGET=linux; TF_TARGET=linux ;; + *) echo "unknown: $OSTYPE" && exit 1;; +esac +fi + +CFLAGS_macbook="-I${TH_TARGET_DIRECTORY}/include" +CFLAGS_linux="-I${TH_TARGET_DIRECTORY}/include" + +var=CFLAGS_${TARGET} +CFLAGS=${!var} +var=LINKER_ARGS_${TARGET} +LINKER_ARGS=${!var} +var=COMPILER_ARGS_${TARGET} +COMPILER_ARGS=${!var} # add -opt for an optimized build. + +mkdir -p $DIR/build/c_interop/ +mkdir -p $DIR/build/bin/ + +cinterop -def $DIR/src/main/c_interop/torch.def -compilerOpts "$CFLAGS" -labels $TARGET \ + -copt -I$TH_TARGET_DIRECTORY/include/TH -o $DIR/build/c_interop/torch || exit 1 + +SOURCE_DIR=$DIR/src/main/kotlin + +konanc $COMPILER_ARGS -target $TARGET $SOURCE_DIR/ClassifierDemo.kt $SOURCE_DIR/Disposable.kt \ + $SOURCE_DIR/Tensors.kt $SOURCE_DIR/Modules.kt $SOURCE_DIR/Dataset.kt $SOURCE_DIR/SmallDemos.kt \ + -library $DIR/build/c_interop/torch \ + -o $DIR/build/bin/HelloTorch \ + -linkerOpts "-L$TH_TARGET_DIRECTORY/lib -lATen" || exit 1 + +echo "Note: You may need to specify LD_LIBRARY_PATH or DYLD_LIBRARY_PATH env variables to $TH_TARGET_DIRECTORY/lib if the ATen dynamic library cannot be found." + +echo "Artifact path is $DIR/build/bin/HelloTorch.kexe" diff --git a/samples/torch/downloadMNIST.sh b/samples/torch/downloadMNIST.sh new file mode 100644 index 00000000000..5519d38510c --- /dev/null +++ b/samples/torch/downloadMNIST.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# See http://yann.lecun.com/exdb/mnist/ + +wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz +wget http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz +wget http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz +wget http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz + +gzip -d train-images-idx3-ubyte.gz +gzip -d train-labels-idx1-ubyte.gz +gzip -d t10k-images-idx3-ubyte.gz +gzip -d t10k-labels-idx1-ubyte.gz \ No newline at end of file diff --git a/samples/torch/downloadTorch.sh b/samples/torch/downloadTorch.sh new file mode 100755 index 00000000000..b8ddee3fd93 --- /dev/null +++ b/samples/torch/downloadTorch.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +TH_TARGET_DIRECTORY=~/.konan/third-party/torch +NO_CUDA=true # set to false for GPU support + +if [ ! -d $TH_TARGET_DIRECTORY/include/THNN ]; then + git clone https://github.com/pytorch/pytorch.git + + mkdir build_torch + cd build_torch + + cmake -DNO_CUDA=$NO_CUDA ../pytorch/aten + make + make DESTDIR=$TH_TARGET_DIRECTORY install + + cd $TH_TARGET_DIRECTORY + + # remove 'usr/local' prefix produced by make: + mv usr/local/* . + rm -d usr/local usr + + # hack to solve "fatal error: 'generic/THNN.h' file not found" when linking, -I$/include/THNN did not work + cp include/THNN/generic/THNN.h include/TH/generic/THNN.h +fi \ No newline at end of file diff --git a/samples/torch/src/main/c_interop/torch.def b/samples/torch/src/main/c_interop/torch.def new file mode 100644 index 00000000000..958c948be70 --- /dev/null +++ b/samples/torch/src/main/c_interop/torch.def @@ -0,0 +1 @@ +headers = TH/TH.h THNN/THNN.h diff --git a/samples/torch/src/main/kotlin/ClassifierDemo.kt b/samples/torch/src/main/kotlin/ClassifierDemo.kt new file mode 100644 index 00000000000..eb63f283a0a --- /dev/null +++ b/samples/torch/src/main/kotlin/ClassifierDemo.kt @@ -0,0 +1,74 @@ +fun Float.toRoundedString(digits: Int = 0): String { + var factor = 1 + + for (i in 0 until digits) { + factor *= 10 + } + + return (kotlin.math.round(this * factor) / factor).toString() +} + +fun Float.toPercentageString(roundToDigits: Int = 1) = (this * 100).toRoundedString(roundToDigits) + +fun List.maxIndex() = withIndex().maxBy { it.value }!!.index + +fun accuracy(predictionBatch: FloatMatrix, labelBatch: FloatMatrix): Float { + val resultIndexes = predictionBatch.toList().map { it.maxIndex() } + val labelBatchIndexes = labelBatch.toList().map { it.maxIndex() } + return resultIndexes.zip(labelBatchIndexes). + count { (result, label) -> result == label }.toFloat() / resultIndexes.size +} + +fun Backpropagatable.trainClassifier( + dataset: Dataset, + lossByLabels: (FloatMatrix) -> Backpropagatable, + learningRateByProgress: (Float) -> Float = { 5f * kotlin.math.exp(-it * 3) }, + batchSize: Int = 64, + iterations: Int = 500) { + + for (i in 0 until iterations) { + disposeScoped { + val (inputBatch, labelBatch) = dataset.sampleBatch(batchSize) + val errorNetwork = this@trainClassifier before lossByLabels(labelBatch) + val forwardResults = use { errorNetwork.forwardPass(inputBatch) } + val accuracy = accuracy(forwardResults.hidden, labelBatch) + val progress = i.toFloat() / iterations + val learningRate = learningRateByProgress(progress) + val backpropResults = use { forwardResults.backpropagate(outputGradient = tensor(learningRate)) } + val crossEntropy = forwardResults.output[0] + backpropResults.descend() + println("Iteration ${i + 1}/$iterations: " + + "${accuracy.toPercentageString()}% training batch accuracy, " + + "cross entropy loss = ${crossEntropy.toRoundedString(4)}, " + + "learning rate = ${learningRate.toRoundedString(4)}") + } + } +} + +fun Backpropagatable.testClassifier(dataset: Dataset, batchSize: Int = 100): Float { + val testBatches = dataset.testBatches(batchSize) + return testBatches.withIndex().map { (i, batchPair) -> + val (inputBatch, outputBatch) = batchPair + val accuracy = accuracy(this.forwardPass(inputBatch).output, outputBatch) + println("Test batch ${i + 1}/${testBatches.size}: ${accuracy.toPercentageString()}% accuracy") + accuracy * inputBatch.shape[0] + }.sum() / dataset.inputs.size +} + +fun randomInit(size: Int) = random(-.01, .01, size) +fun randomInit(size0: Int, size1: Int) = random(-.1, .1, size0, size1) + +fun linear(inputSize: Int, outputSize: Int) = Linear(randomInit(outputSize, inputSize), randomInit(outputSize)) +fun twoLayerClassifier(dataset: Dataset, hiddenSize: Int = 64) = + linear(dataset.inputs[0].size, hiddenSize) before Relu before + linear(hiddenSize, dataset.labels[0].size) before Softmax + +fun main(args: Array) { + val trainingDataset = MNIST.labeledTrainingImages() + val predictionNetwork = twoLayerClassifier(trainingDataset) + predictionNetwork.trainClassifier(trainingDataset, lossByLabels = { CrossEntropyLoss(labels = it) }) + + val testDataset = MNIST.labeledTestImages() + val averageAccuracy = predictionNetwork.testClassifier(testDataset) + println("Accuracy on the test set: ${averageAccuracy.toPercentageString()}") +} \ No newline at end of file diff --git a/samples/torch/src/main/kotlin/Dataset.kt b/samples/torch/src/main/kotlin/Dataset.kt new file mode 100644 index 00000000000..2f97b25d793 --- /dev/null +++ b/samples/torch/src/main/kotlin/Dataset.kt @@ -0,0 +1,96 @@ +import kotlinx.cinterop.* +import platform.posix.* + +data class Dataset(val inputs: List, val labels: List) { + fun batch(indices: List): Pair { + val inputBatch = tensor(*(indices.map { inputs[it].toTypedArray() }.toTypedArray())) + val labelBatch = tensor(*(indices.map { labels[it].toTypedArray() }.toTypedArray())) + return inputBatch to labelBatch + } + + fun sampleBatch(batchSize: Int) = batch((0 until batchSize).map { randomInt(inputs.size) }) + private fun batchAt(batchIndex: Int, batchSize: Int) = + batch((0 until inputs.size).drop(batchSize + batchIndex).take(batchSize)) + + fun testBatches(batchSize: Int) = (0 until inputs.size / batchSize).map { batchAt(it, batchSize = batchSize) } +} + +/** + * Provides the MNIST labeled handwritten digit dataset, described at http://yann.lecun.com/exdb/mnist/ + */ +object MNIST { + private fun readFileData(path: String) = memScoped { + fun fail(): Nothing = throw Error("Cannot read input file $path") + + val size = alloc().also { if (stat(path, it.ptr) != 0) fail() }.st_size.toInt() + + println("Reading $size bytes from $path...") + + val file = fopen(path, "rb") ?: fail() + try { + ByteArray(size).also { fread(it.refTo(0), 1, size.signExtend(), file) } + } finally { + fclose(file) + } + } + + private fun Byte.reinterpretAsUnsigned() = this.toInt().let { it + if (it < 0) 256 else 0 } + + private fun unsignedBytesToInt(bytes: List) = + bytes.withIndex().map { (i, value) -> value.reinterpretAsUnsigned().shl(8 * (3 - i)) }.sum() + + private val intSize = 4 + private fun ByteArray.getIntAt(index: Int) = + unsignedBytesToInt((index until (index + intSize)).map { this[it] }) + + private val imageLength = 28 + private val imageSize = imageLength * imageLength + + private fun ByteArray.getImageAt(index: Int) = + FloatArray(imageSize) { this[index + it].reinterpretAsUnsigned().toFloat() / 255 } + + private fun oneHot(size: Int, index: Int) = FloatArray(size) { if (it == index) 1f else 0f } + + private fun readLabels(filePath: String, totalLabels: Int = 10): List { + val data = readFileData(filePath) + val check = data.getIntAt(0) + val expectedCheck = 2049 + if (check != 2049) throw Error("File should start with int $expectedCheck, but was $check.") + + val count = data.getIntAt(4) + + val offset = 8 + + if (count + offset != data.size) throw Error("Unexpected file size: ${data.size}.") + + return (0 until count).map { oneHot(totalLabels, index = data[offset + it].reinterpretAsUnsigned()) } + } + + private fun readImages(filePath: String): List { + val data = readFileData(filePath) + val check = data.getIntAt(0) + val expectedCheck = 2051 + if (check != expectedCheck) throw Error("File should start with int $expectedCheck, but was $check.") + + val count = data.getIntAt(4) + val width = data.getIntAt(8) + val height = data.getIntAt(12) + + val offset = 16 + + if (width != imageLength) throw Error() + if (height != imageLength) throw Error() + + if (count * imageSize + offset != data.size) throw Error("Unexpected file size: ${data.size}.") + + return (0 until count).map { data.getImageAt(offset + imageSize * it) } + } + + fun labeledTrainingImages() = Dataset( + inputs = readImages("train-images-idx3-ubyte"), + labels = readLabels("train-labels-idx1-ubyte")) + + fun labeledTestImages() = Dataset( + inputs = readImages("t10k-images-idx3-ubyte"), + labels = readLabels("t10k-labels-idx1-ubyte")) +} \ No newline at end of file diff --git a/samples/torch/src/main/kotlin/Disposable.kt b/samples/torch/src/main/kotlin/Disposable.kt new file mode 100644 index 00000000000..e29e66b1238 --- /dev/null +++ b/samples/torch/src/main/kotlin/Disposable.kt @@ -0,0 +1,26 @@ +interface Disposable { + fun dispose() +} + +open class DisposableContainer(private val disposables: MutableList = ArrayList()) : Disposable { + /** + * Creates the object and schedules its disposal for the end of the scope. + */ + fun use(create: () -> T) = create().also { disposables.add(it) } + + override fun dispose() { + for (disposable in disposables) { + disposable.dispose() + } + } +} + +fun disposeScoped(action: DisposableContainer.() -> T): T { + val scope = DisposableContainer() + + try { + return scope.action() + } finally { + scope.dispose() + } +} \ No newline at end of file diff --git a/samples/torch/src/main/kotlin/Modules.kt b/samples/torch/src/main/kotlin/Modules.kt new file mode 100644 index 00000000000..949e8d2a66b --- /dev/null +++ b/samples/torch/src/main/kotlin/Modules.kt @@ -0,0 +1,205 @@ +import kotlinx.cinterop.* +import torch.* + +// Defines network modules with the ability for backpropagation using both TH.h and THNN.h from the ATen library + +abstract class Backpropagatable { + abstract inner class ForwardResults(val input: Input) : DisposableContainer() { + init { + use { input } + } + + abstract val output: Output + abstract fun backpropagate(outputGradient: Output): BackpropagationResults + } + + abstract inner class BackpropagationResults( + val input: Input, + val output: Output, + val outputGradient: Output + ) : DisposableContainer() { + init { + use { input } + use { output } + use { outputGradient } + } + + abstract val inputGradient: Input + abstract fun descend() + } + + abstract fun forwardPass(input: Input): ForwardResults +} + +abstract class Module : Backpropagatable() { + abstract var parameters: Parameters + abstract fun parametersToList(parameters: Parameters): List + abstract fun parametersFromList(list: List): Parameters + private val parameterList get() = parametersToList(parameters) + + abstract operator fun invoke(input: Input): Output + abstract fun inputGradient(input: Input, outputGradient: Output, output: Output): Input + abstract fun parameterGradient(input: Input, outputGradient: Output, inputGradient: Input): Parameters + + override fun forwardPass(input: Input) = object : ForwardResults(input) { + override val output = use { this@Module(input) } + override fun backpropagate(outputGradient: Output) = + object : Backpropagatable.BackpropagationResults(input, output, outputGradient) { + override val inputGradient = use { this@Module.inputGradient(input, outputGradient, output) } + val parameterGradient = this@Module.parameterGradient(input, + outputGradient = outputGradient, inputGradient = inputGradient) + + override fun descend() = this@Module.descend(parameterGradient) + } + } + + open fun descend(parameterGradient: Parameters) { + parameters = parametersFromList(parameterList.zip( + parametersToList(parameterGradient)) { parameter, gradient -> parameter - gradient }) + } +} + +abstract class ParameterFreeModule : Module() { + override var parameters = Unit + override fun parametersToList(parameters: Unit) = emptyList() + override fun parametersFromList(list: List) = Unit + override fun parameterGradient(input: Input, outputGradient: Output, inputGradient: Input) = Unit + override fun descend(parameterGradient: Unit) {} +} + +class Chain( + val module1: Backpropagatable