Add Torch example (#1209)

2018-01-16 11:49:47 +00:00
parent 45cd41ce81
commit 0c3d5c52fc
13 changed files with 774 additions and 0 deletions
@@ -10,6 +10,7 @@ include ':opengl'
 include ':socket'
 include ':tetris'
 include ':tensorflow'
+include ':torch'
 // Android native activity build requires Android SDK. 
 // So temporary switching off for now, as it breaks the build 
 // of other samples if SDK is not present.
@@ -0,0 +1,47 @@
+# Torch demo
+
+Trains a handwritten digit classifier using the [Torch](http://torch.ch) C backend.
+Like other Torch clients, most prominently [PyTorch](http://pytorch.org),
+this example is built on top of the 
+[ATen C API](https://github.com/pytorch/pytorch/tree/master/aten),
+showing how a Torch client for Kotlin/Native could look like.
+
+## Installation
+
+To build [ATen (Torch for C)](https://github.com/pytorch/pytorch/tree/master/aten), 
+make sure you have Python 2.X and pyyaml installed:
+
+    # macOS: if you don't have pip
+    sudo easy_install pip
+    # Linux: if you don't have pip
+    apt-get -y install python-pip
+    
+    # if you don't have pyyaml
+    sudo pip install pyyaml
+    
+Now
+
+    ./downloadTorch.sh
+
+will install it into `$HOME/.konan/third-party/torch` (if not yet done). 
+
+To build use `../gradlew build` or `./build.sh`.
+
+    ./downloadMNIST.sh
+
+will download and unzip the [MNIST dataset](https://en.wikipedia.org/wiki/MNIST_database) of 
+[70000 labeled handwritten digits](http://yann.lecun.com/exdb/mnist/) for training and testing a classifier.
+
+Then run 
+
+    ../gradlew run
+    
+Alternatively you can run the artifact directly through
+
+    ./build/konan/bin/macbook/HelloTorch.kexe
+    
+You may need to specify `LD_LIBRARY_PATH` or `DYLD_LIBRARY_PATH` to `$HOME/.konan/third-party/torch/lib`
+if the ATen dynamic library cannot be found.
+
+Even on a CPU, training should only take some minutes, 
+and you should observe a classification accuracy of about 95% on the test dataset.
@@ -0,0 +1,34 @@
+apply plugin: 'konan'
+
+konan.targets = ['macbook', 'linux']
+
+def torchHome = "${System.getProperty("user.home")}/.konan/third-party/torch"
+
+task downloadTorch(type: Exec) {
+    workingDir getProjectDir()
+    commandLine './downloadTorch.sh'
+}
+
+konanArtifacts {
+    interop('TorchInterop') {
+        defFile "src/main/c_interop/torch.def"
+        includeDirs "${torchHome}/include", "${torchHome}/include/TH"
+        dependsOn 'downloadTorch'
+    }
+
+    program('Torch') {
+        libraries {
+            artifact 'TorchInterop'
+        }
+        linkerOpts "-L${torchHome}/lib -lATen"
+    }
+}
+
+run.dependsOn 'warning'
+
+task warning {
+    doLast {
+        println "Note: You may need to specify LD_LIBRARY_PATH or DYLD_LIBRARY_PATH env variables to $torchHome/lib if the ATen dynamic library cannot be found."
+
+    }
+}
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+DIR=$(cd "$(dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+source "$DIR/../konan.sh"
+
+$DIR/downloadTorch.sh
+
+TH_TARGET_DIRECTORY="$HOME/.konan/third-party/torch"
+
+if [ x$TARGET == x ]; then
+case "$OSTYPE" in
+  darwin*)  TARGET=macbook; TF_TARGET=darwin ;;
+  linux*)   TARGET=linux; TF_TARGET=linux ;;
+  *)        echo "unknown: $OSTYPE" && exit 1;;
+esac
+fi
+
+CFLAGS_macbook="-I${TH_TARGET_DIRECTORY}/include"
+CFLAGS_linux="-I${TH_TARGET_DIRECTORY}/include"
+
+var=CFLAGS_${TARGET}
+CFLAGS=${!var}
+var=LINKER_ARGS_${TARGET}
+LINKER_ARGS=${!var}
+var=COMPILER_ARGS_${TARGET}
+COMPILER_ARGS=${!var} # add -opt for an optimized build.
+
+mkdir -p $DIR/build/c_interop/
+mkdir -p $DIR/build/bin/
+
+cinterop -def $DIR/src/main/c_interop/torch.def -compilerOpts "$CFLAGS" -labels $TARGET \
+     -copt -I$TH_TARGET_DIRECTORY/include/TH -o $DIR/build/c_interop/torch || exit 1
+
+SOURCE_DIR=$DIR/src/main/kotlin
+
+konanc $COMPILER_ARGS -target $TARGET $SOURCE_DIR/ClassifierDemo.kt $SOURCE_DIR/Disposable.kt \
+       $SOURCE_DIR/Tensors.kt $SOURCE_DIR/Modules.kt $SOURCE_DIR/Dataset.kt $SOURCE_DIR/SmallDemos.kt \
+       -library $DIR/build/c_interop/torch \
+       -o $DIR/build/bin/HelloTorch \
+       -linkerOpts "-L$TH_TARGET_DIRECTORY/lib -lATen" || exit 1
+
+echo "Note: You may need to specify LD_LIBRARY_PATH or DYLD_LIBRARY_PATH env variables to $TH_TARGET_DIRECTORY/lib if the ATen dynamic library cannot be found."
+
+echo "Artifact path is $DIR/build/bin/HelloTorch.kexe"
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# See http://yann.lecun.com/exdb/mnist/
+
+wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz
+wget http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz
+wget http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz
+wget http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz
+
+gzip -d train-images-idx3-ubyte.gz
+gzip -d train-labels-idx1-ubyte.gz
+gzip -d t10k-images-idx3-ubyte.gz
+gzip -d t10k-labels-idx1-ubyte.gz
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+TH_TARGET_DIRECTORY=~/.konan/third-party/torch
+NO_CUDA=true # set to false for GPU support
+
+if [ ! -d $TH_TARGET_DIRECTORY/include/THNN ]; then
+    git clone https://github.com/pytorch/pytorch.git
+
+    mkdir build_torch
+    cd build_torch
+
+    cmake -DNO_CUDA=$NO_CUDA ../pytorch/aten
+    make
+    make DESTDIR=$TH_TARGET_DIRECTORY install
+
+    cd $TH_TARGET_DIRECTORY
+
+    # remove 'usr/local' prefix produced by make:
+    mv usr/local/* .
+    rm -d usr/local usr
+
+    # hack to solve "fatal error: 'generic/THNN.h' file not found" when linking, -I$<DIR>/include/THNN did not work
+    cp include/THNN/generic/THNN.h include/TH/generic/THNN.h
+fi
@@ -0,0 +1 @@
+headers = TH/TH.h THNN/THNN.h
@@ -0,0 +1,74 @@
+fun Float.toRoundedString(digits: Int = 0): String {
+    var factor = 1
+
+    for (i in 0 until digits) {
+        factor *= 10
+    }
+
+    return (kotlin.math.round(this * factor) / factor).toString()
+}
+
+fun Float.toPercentageString(roundToDigits: Int = 1) = (this * 100).toRoundedString(roundToDigits)
+
+fun List<Float>.maxIndex() = withIndex().maxBy { it.value }!!.index
+
+fun accuracy(predictionBatch: FloatMatrix, labelBatch: FloatMatrix): Float {
+    val resultIndexes = predictionBatch.toList().map { it.maxIndex() }
+    val labelBatchIndexes = labelBatch.toList().map { it.maxIndex() }
+    return resultIndexes.zip(labelBatchIndexes).
+            count { (result, label) -> result == label }.toFloat() / resultIndexes.size
+}
+
+fun Backpropagatable<FloatMatrix, FloatMatrix>.trainClassifier(
+        dataset: Dataset,
+        lossByLabels: (FloatMatrix) -> Backpropagatable<FloatMatrix, FloatVector>,
+        learningRateByProgress: (Float) -> Float = { 5f * kotlin.math.exp(-it * 3) },
+        batchSize: Int = 64,
+        iterations: Int = 500) {
+
+    for (i in 0 until iterations) {
+        disposeScoped {
+            val (inputBatch, labelBatch) = dataset.sampleBatch(batchSize)
+            val errorNetwork = this@trainClassifier before lossByLabels(labelBatch)
+            val forwardResults = use { errorNetwork.forwardPass(inputBatch) }
+            val accuracy = accuracy(forwardResults.hidden, labelBatch)
+            val progress = i.toFloat() / iterations
+            val learningRate = learningRateByProgress(progress)
+            val backpropResults = use { forwardResults.backpropagate(outputGradient = tensor(learningRate)) }
+            val crossEntropy = forwardResults.output[0]
+            backpropResults.descend()
+            println("Iteration ${i + 1}/$iterations: " +
+                    "${accuracy.toPercentageString()}% training batch accuracy, " +
+                    "cross entropy loss = ${crossEntropy.toRoundedString(4)}, " +
+                    "learning rate = ${learningRate.toRoundedString(4)}")
+        }
+    }
+}
+
+fun Backpropagatable<FloatMatrix, FloatMatrix>.testClassifier(dataset: Dataset, batchSize: Int = 100): Float {
+    val testBatches = dataset.testBatches(batchSize)
+    return testBatches.withIndex().map { (i, batchPair) ->
+        val (inputBatch, outputBatch) = batchPair
+        val accuracy = accuracy(this.forwardPass(inputBatch).output, outputBatch)
+        println("Test batch ${i + 1}/${testBatches.size}: ${accuracy.toPercentageString()}% accuracy")
+        accuracy * inputBatch.shape[0]
+    }.sum() / dataset.inputs.size
+}
+
+fun randomInit(size: Int) = random(-.01, .01, size)
+fun randomInit(size0: Int, size1: Int) = random(-.1, .1, size0, size1)
+
+fun linear(inputSize: Int, outputSize: Int) = Linear(randomInit(outputSize, inputSize), randomInit(outputSize))
+fun twoLayerClassifier(dataset: Dataset, hiddenSize: Int = 64) =
+        linear(dataset.inputs[0].size, hiddenSize) before Relu before
+                linear(hiddenSize, dataset.labels[0].size) before Softmax
+
+fun main(args: Array<String>) {
+    val trainingDataset = MNIST.labeledTrainingImages()
+    val predictionNetwork = twoLayerClassifier(trainingDataset)
+    predictionNetwork.trainClassifier(trainingDataset, lossByLabels = { CrossEntropyLoss(labels = it) })
+
+    val testDataset = MNIST.labeledTestImages()
+    val averageAccuracy = predictionNetwork.testClassifier(testDataset)
+    println("Accuracy on the test set: ${averageAccuracy.toPercentageString()}")
+}
@@ -0,0 +1,96 @@
+import kotlinx.cinterop.*
+import platform.posix.*
+
+data class Dataset(val inputs: List<FloatArray>, val labels: List<FloatArray>) {
+    fun batch(indices: List<Int>): Pair<FloatMatrix, FloatMatrix> {
+        val inputBatch = tensor(*(indices.map { inputs[it].toTypedArray() }.toTypedArray()))
+        val labelBatch = tensor(*(indices.map { labels[it].toTypedArray() }.toTypedArray()))
+        return inputBatch to labelBatch
+    }
+
+    fun sampleBatch(batchSize: Int) = batch((0 until batchSize).map { randomInt(inputs.size) })
+    private fun batchAt(batchIndex: Int, batchSize: Int) =
+            batch((0 until inputs.size).drop(batchSize + batchIndex).take(batchSize))
+
+    fun testBatches(batchSize: Int) = (0 until inputs.size / batchSize).map { batchAt(it, batchSize = batchSize) }
+}
+
+/**
+ * Provides the MNIST labeled handwritten digit dataset, described at http://yann.lecun.com/exdb/mnist/
+ */
+object MNIST {
+    private fun readFileData(path: String) = memScoped {
+        fun fail(): Nothing = throw Error("Cannot read input file $path")
+
+        val size = alloc<stat>().also { if (stat(path, it.ptr) != 0) fail() }.st_size.toInt()
+
+        println("Reading $size bytes from $path...")
+
+        val file = fopen(path, "rb") ?: fail()
+        try {
+            ByteArray(size).also { fread(it.refTo(0), 1, size.signExtend(), file) }
+        } finally {
+            fclose(file)
+        }
+    }
+
+    private fun Byte.reinterpretAsUnsigned() = this.toInt().let { it + if (it < 0) 256 else 0 }
+
+    private fun unsignedBytesToInt(bytes: List<Byte>) =
+            bytes.withIndex().map { (i, value) -> value.reinterpretAsUnsigned().shl(8 * (3 - i)) }.sum()
+
+    private val intSize = 4
+    private fun ByteArray.getIntAt(index: Int) =
+            unsignedBytesToInt((index until (index + intSize)).map { this[it] })
+
+    private val imageLength = 28
+    private val imageSize = imageLength * imageLength
+
+    private fun ByteArray.getImageAt(index: Int) =
+            FloatArray(imageSize) { this[index + it].reinterpretAsUnsigned().toFloat() / 255 }
+
+    private fun oneHot(size: Int, index: Int) = FloatArray(size) { if (it == index) 1f else 0f }
+
+    private fun readLabels(filePath: String, totalLabels: Int = 10): List<FloatArray> {
+        val data = readFileData(filePath)
+        val check = data.getIntAt(0)
+        val expectedCheck = 2049
+        if (check != 2049) throw Error("File should start with int $expectedCheck, but was $check.")
+
+        val count = data.getIntAt(4)
+
+        val offset = 8
+
+        if (count + offset != data.size) throw Error("Unexpected file size: ${data.size}.")
+
+        return (0 until count).map { oneHot(totalLabels, index = data[offset + it].reinterpretAsUnsigned()) }
+    }
+
+    private fun readImages(filePath: String): List<FloatArray> {
+        val data = readFileData(filePath)
+        val check = data.getIntAt(0)
+        val expectedCheck = 2051
+        if (check != expectedCheck) throw Error("File should start with int $expectedCheck, but was $check.")
+
+        val count = data.getIntAt(4)
+        val width = data.getIntAt(8)
+        val height = data.getIntAt(12)
+
+        val offset = 16
+
+        if (width != imageLength) throw Error()
+        if (height != imageLength) throw Error()
+
+        if (count * imageSize + offset != data.size) throw Error("Unexpected file size: ${data.size}.")
+
+        return (0 until count).map { data.getImageAt(offset + imageSize * it) }
+    }
+
+    fun labeledTrainingImages() = Dataset(
+            inputs = readImages("train-images-idx3-ubyte"),
+            labels = readLabels("train-labels-idx1-ubyte"))
+
+    fun labeledTestImages() = Dataset(
+            inputs = readImages("t10k-images-idx3-ubyte"),
+            labels = readLabels("t10k-labels-idx1-ubyte"))
+}
@@ -0,0 +1,26 @@
+interface Disposable {
+    fun dispose()
+}
+
+open class DisposableContainer(private val disposables: MutableList<Disposable> = ArrayList()) : Disposable {
+    /**
+     * Creates the object and schedules its disposal for the end of the scope.
+     */
+    fun <T : Disposable> use(create: () -> T) = create().also { disposables.add(it) }
+
+    override fun dispose() {
+        for (disposable in disposables) {
+            disposable.dispose()
+        }
+    }
+}
+
+fun <T> disposeScoped(action: DisposableContainer.() -> T): T {
+    val scope = DisposableContainer()
+
+    try {
+        return scope.action()
+    } finally {
+        scope.dispose()
+    }
+}
@@ -0,0 +1,205 @@
+import kotlinx.cinterop.*
+import torch.*
+
+// Defines network modules with the ability for backpropagation using both TH.h and THNN.h from the ATen library
+
+abstract class Backpropagatable<Input : Disposable, Output : Disposable> {
+    abstract inner class ForwardResults(val input: Input) : DisposableContainer() {
+        init {
+            use { input }
+        }
+
+        abstract val output: Output
+        abstract fun backpropagate(outputGradient: Output): BackpropagationResults
+    }
+
+    abstract inner class BackpropagationResults(
+            val input: Input,
+            val output: Output,
+            val outputGradient: Output
+    ) : DisposableContainer() {
+        init {
+            use { input }
+            use { output }
+            use { outputGradient }
+        }
+
+        abstract val inputGradient: Input
+        abstract fun descend()
+    }
+
+    abstract fun forwardPass(input: Input): ForwardResults
+}
+
+abstract class Module<Input : Disposable, Output : Disposable, Parameters> : Backpropagatable<Input, Output>() {
+    abstract var parameters: Parameters
+    abstract fun parametersToList(parameters: Parameters): List<FloatTensor>
+    abstract fun parametersFromList(list: List<FloatTensor>): Parameters
+    private val parameterList get() = parametersToList(parameters)
+
+    abstract operator fun invoke(input: Input): Output
+    abstract fun inputGradient(input: Input, outputGradient: Output, output: Output): Input
+    abstract fun parameterGradient(input: Input, outputGradient: Output, inputGradient: Input): Parameters
+
+    override fun forwardPass(input: Input) = object : ForwardResults(input) {
+        override val output = use { this@Module(input) }
+        override fun backpropagate(outputGradient: Output) =
+                object : Backpropagatable<Input, Output>.BackpropagationResults(input, output, outputGradient) {
+                    override val inputGradient = use { this@Module.inputGradient(input, outputGradient, output) }
+                    val parameterGradient = this@Module.parameterGradient(input,
+                            outputGradient = outputGradient, inputGradient = inputGradient)
+
+                    override fun descend() = this@Module.descend(parameterGradient)
+                }
+    }
+
+    open fun descend(parameterGradient: Parameters) {
+        parameters = parametersFromList(parameterList.zip(
+                parametersToList(parameterGradient)) { parameter, gradient -> parameter - gradient })
+    }
+}
+
+abstract class ParameterFreeModule<Input : Disposable, Output : Disposable> : Module<Input, Output, Unit>() {
+    override var parameters = Unit
+    override fun parametersToList(parameters: Unit) = emptyList<FloatTensor>()
+    override fun parametersFromList(list: List<FloatTensor>) = Unit
+    override fun parameterGradient(input: Input, outputGradient: Output, inputGradient: Input) = Unit
+    override fun descend(parameterGradient: Unit) {}
+}
+
+class Chain<Input : Disposable, Hidden : Disposable, Output : Disposable>(
+        val module1: Backpropagatable<Input, Hidden>,
+        val module2: Backpropagatable<Hidden, Output>
+) : Backpropagatable<Input, Output>() {
+    override fun forwardPass(input: Input) = ChainForwardResults(input)
+
+    inner class ChainForwardResults(input: Input) : ForwardResults(input) {
+        val result1 = use { module1.forwardPass(input) }
+        val hidden = result1.output
+        val result2 = use { module2.forwardPass(result1.output) }
+        override val output = result2.output
+        override fun backpropagate(outputGradient: Output) =
+                object : Backpropagatable<Input, Output>.BackpropagationResults(input, output, outputGradient) {
+                    val backpropResults2 = use { result2.backpropagate(outputGradient) }
+                    val hiddenGradient = backpropResults2.inputGradient
+                    val backpropResults1 = use { result1.backpropagate(hiddenGradient) }
+
+                    override val inputGradient = backpropResults1.inputGradient
+
+                    override fun descend() {
+                        backpropResults1.descend()
+                        backpropResults2.descend()
+                    }
+                }
+    }
+
+    override fun toString() = "$module1 before $module2"
+}
+
+infix fun <Input : Disposable, Hidden : Disposable, Output : Disposable> Backpropagatable<Input, Hidden>.before(
+        other: Backpropagatable<Hidden, Output>) = Chain(this, other)
+
+object Abs : ParameterFreeModule<FloatMatrix, FloatMatrix>() {
+    override operator fun invoke(input: FloatMatrix) = initializedTensor(input.shape[0], input.shape[1]) {
+        THNN_FloatAbs_updateOutput(cValuesOf<FloatVar>(), input.raw, it.raw)
+    }
+
+    override fun inputGradient(input: FloatMatrix, outputGradient: FloatMatrix, output: FloatMatrix) =
+            initializedTensor(input.shape[0], input.shape[1]) {
+                THNN_FloatAbs_updateGradInput(null, input.raw, outputGradient.raw, it.raw)
+            }
+}
+
+object Relu : ParameterFreeModule<FloatMatrix, FloatMatrix>() {
+    override operator fun invoke(input: FloatMatrix) = initializedTensor(input.shape[0], input.shape[1]) {
+        THNN_FloatLeakyReLU_updateOutput(null, input.raw, it.raw, 0.0, false)
+    }
+
+    override fun inputGradient(input: FloatMatrix, outputGradient: FloatMatrix, output: FloatMatrix) =
+            initializedTensor(input.shape[0], input.shape[1]) {
+                THNN_FloatLeakyReLU_updateGradInput(null, input.raw, outputGradient.raw, it.raw, 0.0, false)
+            }
+}
+
+object Softmax : ParameterFreeModule<FloatMatrix, FloatMatrix>() {
+    override operator fun invoke(input: FloatMatrix) = initializedTensor(input.shape[0], input.shape[1]) {
+        THNN_FloatSoftMax_updateOutput(null, input.raw, it.raw, 1)
+    }
+
+    override fun inputGradient(input: FloatMatrix, outputGradient: FloatMatrix, output: FloatMatrix) =
+            initializedTensor(input.shape[0], input.shape[1]) {
+                THNN_FloatSoftMax_updateGradInput(null, input.raw, outputGradient.raw, it.raw, output.raw, 1)
+            }
+}
+
+class MeanSquaredError(val labels: FloatMatrix) : ParameterFreeModule<FloatMatrix, FloatVector>() {
+    override operator fun invoke(input: FloatMatrix) = initializedTensor(1) {
+        THNN_FloatMSECriterion_updateOutput(null, input.raw, labels.raw, it.raw,
+                sizeAverage = true, reduce = true)
+    }
+
+    override fun inputGradient(
+            input: FloatMatrix,
+            outputGradient: FloatVector,
+            output: FloatVector
+    ) = initializedTensor(input.shape[0], input.shape[1]) {
+        THNN_FloatMSECriterion_updateGradInput(null, input.raw, labels.raw,
+                outputGradient.raw, it.raw, sizeAverage = true, reduce = true)
+    }
+}
+
+class CrossEntropyLoss(val labels: FloatMatrix) : ParameterFreeModule<FloatMatrix, FloatVector>() {
+    override operator fun invoke(input: FloatMatrix) = initializedTensor(1) {
+        THNN_FloatBCECriterion_updateOutput(null, input.raw, labels.raw, it.raw,
+                sizeAverage = true, reduce = true, weights = null)
+    }
+
+    override fun inputGradient(input: FloatMatrix, outputGradient: FloatVector, output: FloatVector) =
+            initializedTensor(input.shape[0], input.shape[1]) {
+                THNN_FloatBCECriterion_updateGradInput(null, input.raw, labels.raw, outputGradient.raw,
+                        it.raw, sizeAverage = true, reduce = true, weights = null)
+            }
+}
+
+data class Linear(
+        var weight: FloatMatrix,
+        var bias: FloatVector) : Module<FloatMatrix, FloatMatrix, Pair<FloatMatrix, FloatVector>>() {
+    val inputSize = weight.shape[1]
+    val outputSize = weight.shape[0]
+    val addBuffer = uninitializedTensor(outputSize)
+
+    override operator fun invoke(input: FloatMatrix) = initializedTensor(input.shape[0], outputSize) {
+        THNN_FloatLinear_updateOutput(null, input.raw, it.raw, weight.raw, bias.raw, addBuffer.raw)
+    }
+
+    override fun inputGradient(input: FloatMatrix, outputGradient: FloatMatrix, output: FloatMatrix) =
+            initializedTensor(input.shape[0], inputSize) {
+                THNN_FloatLinear_updateGradInput(null, input.raw, outputGradient.raw, it.raw, weight.raw)
+            }
+
+    override fun parameterGradient(
+            input: FloatMatrix,
+            outputGradient: FloatMatrix,
+            inputGradient: FloatMatrix
+    ): Pair<FloatMatrix, FloatVector> {
+        val biasGradient = zeros(outputSize)
+        val weightGradient = zeros(weight.shape[0], weight.shape[1]).also {
+            THNN_FloatLinear_accGradParameters(null, input.raw, outputGradient.raw, inputGradient.raw, weight.raw,
+                    bias.raw, it.raw, biasGradient.raw, addBuffer.raw, 1.0)
+        }
+
+        return weightGradient to biasGradient
+    }
+
+    override var parameters: Pair<FloatMatrix, FloatVector>
+        get() = weight to bias
+        set(value) {
+            weight = value.first
+            bias = value.second
+        }
+
+    override fun parametersToList(parameters: Pair<FloatMatrix, FloatVector>) =
+            listOf(parameters.first, parameters.second)
+
+    override fun parametersFromList(list: List<FloatTensor>) = list.first().asMatrix() to list.last().asVector()
+}
@@ -0,0 +1,55 @@
+// If you are curious you can also try one of these
+
+private fun demonstrateTensors() {
+    disposeScoped {
+        val x = use { tensor(0f, 1f, 2f) }
+        val y = use { tensor(0f, -1f, -2f) }
+        val m = use {
+            tensor(
+                    arrayOf(1f, -1f, 0f),
+                    arrayOf(0f, -1f, 0f),
+                    arrayOf(0f, 0f, -.5f))
+        }
+
+        println("Hello, Torch!\nx = $x\ny = $y\n" +
+                "|x| = ${x.abs()}\n|y| = ${y.abs()}\n" +
+                "2x=${use { x * 2f }}\nx+y = ${use { x + y }}\nx-y = ${use { x - y }}\nxy = ${x * y}\n" +
+                "m=\n${use { m }}\nm·y = ${use { m * y }}\nm+m =\n${use { m + m }}\nm·m =\n${use { m * m }}")
+    }
+}
+
+private fun demonstrateModules() {
+    val input = tensor(arrayOf(-1f))
+    val abs = Abs(input)
+    println("abs of $input is $abs, gradient is ${Abs.inputGradient(input, tensor(arrayOf(1f)), abs)}")
+    val relu = Relu(input)
+    println("relu of $input is $relu, gradient is ${Relu.inputGradient(input, tensor(arrayOf(1f)), relu)}")
+}
+
+private fun demonstrateManualBackpropagationFor1LinearLayer(
+        inputs: FloatMatrix = tensor(arrayOf(1f, -1f), arrayOf(1f, -1f)),
+        labels: FloatMatrix = tensor(arrayOf(5f), arrayOf(5f)),
+        learningRate: Float = .1f) {
+    val linear = Linear(weight = randomInit(1, 2), bias = randomInit(1))
+    val error = MeanSquaredError(labels)
+
+    for (i in 0 until 100) {
+        disposeScoped {
+            val output = use { linear(inputs) }
+            val loss = use { error(output) }
+            val outputGradient = use { error.inputGradient(output, tensor(learningRate), loss) }
+            val inputGradient = use { linear.inputGradient(inputs, outputGradient, output) }
+            val parameterGradient = linear.parameterGradient(inputs, outputGradient, inputGradient).
+                    also { use { it.first } }.also { use { it.second } }
+            println("input: $inputs, \n" +
+                    "output: $output, \n" +
+                    "labels: $labels, \n" +
+                    "mean squared error: $loss, \n" +
+                    "output gradient: $outputGradient, \n" +
+                    "input gradient: $inputGradient, \n" +
+                    "parameter gradient: $parameterGradient")
+            linear.weight -= parameterGradient.first
+            linear.bias -= parameterGradient.second
+        }
+    }
+}
@@ -0,0 +1,153 @@
+import kotlinx.cinterop.*
+import torch.*
+
+// Defines tensor classes and operations using TH.h from the ATen library
+
+abstract class FloatTensor(val raw: CPointer<THFloatTensor>) : Disposable {
+    private val storage: CPointer<THFloatStorage> get() = raw.pointed.storage!!
+    private val elements get() = storage.pointed
+    private val data: CPointer<FloatVar> get() = elements.data!!
+    private val size: CPointer<LongVar> get() = raw.pointed.size!!
+    protected val nDimension: Int get() = raw.pointed.nDimension
+
+    val shape: List<Int> get() = (0 until nDimension).map { size[it].toInt() }
+
+    operator fun plus(other: FloatTensor) = initializedTensor(shape) {
+        THFloatTensor_cadd(it.raw, raw, 1f, other.raw)
+    }
+
+    operator fun minus(other: FloatTensor) = initializedTensor(shape) {
+        THFloatTensor_cadd(it.raw, raw, -1f, other.raw)
+    }
+
+    open operator fun times(factor: Float) = initializedTensor(shape) {
+        THFloatTensor_mul(it.raw, raw, factor)
+    }
+
+    fun sum() = THFloatTensor_sumall(raw)
+    fun flatten() = (0 until elements.size).map { data[it] }.toTypedArray()
+
+    override fun dispose() {
+        THFloatTensor_free(raw)
+    }
+
+    fun asVector() = FloatVector(raw)
+    fun asMatrix() = FloatMatrix(raw)
+    inline fun <reified T : FloatTensor> asTensor(): T = when (T::class) {
+        FloatVector::class -> asVector() as T
+        FloatMatrix::class -> asMatrix() as T
+        FloatTensor::class -> this as T
+        else -> throw Error("Unexpected class ${T::class}")
+    }
+
+    abstract override fun toString(): String
+}
+
+class FloatVector(raw: CPointer<THFloatTensor>) : FloatTensor(raw) {
+    init {
+        if (super.nDimension != 1)
+            throw Error("A vector must have exactly 1 dimension.")
+    }
+
+    operator fun get(i: Int) = THFloatTensor_get1d(raw, i.signExtend())
+    operator fun set(i: Int, value: Float) = THFloatTensor_set1d(raw, i.signExtend(), value)
+    fun toArray() = (0 until shape[0]).map { i0 -> this[i0] }.toTypedArray()
+
+    operator fun plus(other: FloatVector) = super.plus(other).asVector()
+    operator fun minus(other: FloatVector) = super.minus(other).asVector()
+    override operator fun times(factor: Float) = super.times(factor).asVector()
+    operator fun times(other: FloatVector) = THFloatTensor_dot(raw, other.raw)
+
+    fun abs() = kotlin.math.sqrt(this * this)
+
+    override fun toString() = "[${toArray().joinToString { it.toString() }}]"
+}
+
+class FloatMatrix(raw: CPointer<THFloatTensor>) : FloatTensor(raw) {
+    init {
+        if (super.nDimension != 2)
+            throw Error("A matrix must have exactly 2 dimensions.")
+    }
+
+    fun getRow(i0: Int) = (0 until shape[1]).map { i1 -> this[i0, i1] }
+    operator fun get(i0: Int, i1: Int) = THFloatTensor_get2d(raw, i0.signExtend(), i1.signExtend())
+    operator fun set(i0: Int, i1: Int, value: Float) = THFloatTensor_set2d(raw, i0.signExtend(), i1.signExtend(), value)
+    fun toList() = (0 until shape[0]).map { getRow(it) }
+
+    operator fun plus(other: FloatMatrix) = super.plus(other).asMatrix()
+    operator fun minus(other: FloatMatrix) = super.minus(other).asMatrix()
+    override operator fun times(factor: Float) = super.times(factor).asMatrix()
+
+    operator fun times(vector: FloatVector) = initializedTensor(shape[0]) {
+        THFloatTensor_addmv(it.raw, 0f, it.raw, 1f, raw, vector.raw)
+    }
+
+    operator fun times(matrix: FloatMatrix) = initializedTensor(shape[0], matrix.shape[1]) {
+        THFloatTensor_addmm(it.raw, 0f, it.raw, 1f, raw, matrix.raw)
+    }
+
+    override fun toString() = "[${toList().joinToString(",\n") { "[${it.joinToString { it.toString() }}]" }}]"
+}
+
+fun uninitializedTensor(size: Int) =
+        FloatVector(THFloatTensor_newWithSize1d(size.signExtend())!!)
+
+fun uninitializedTensor(size0: Int, size1: Int) =
+        FloatMatrix(THFloatTensor_newWithSize2d(size0.signExtend(), size1.signExtend())!!)
+
+fun uninitializedTensor(shape: List<Int>) = when (shape.size) {
+    1 -> uninitializedTensor(shape.single())
+    2 -> uninitializedTensor(shape[0], shape[1])
+    else -> throw Error("Tensors with ${shape.size} dimensions are not supported yet.")
+}
+
+fun <T> initializedTensor(size: Int, initializer: (FloatVector) -> T) =
+        uninitializedTensor(size).apply { initializer(this) }
+
+fun <T> initializedTensor(size0: Int, size1: Int, initializer: (FloatMatrix) -> T) =
+        uninitializedTensor(size0, size1).apply { initializer(this) }
+
+fun <T> initializedTensor(shape: List<Int>, initializer: (FloatTensor) -> T) =
+        uninitializedTensor(shape).apply { initializer(this) }
+
+fun tensor(size: Int, initializer: (Int) -> Float) = initializedTensor(size) {
+    for (i in 0 until size) {
+        it[i] = initializer(i)
+    }
+}
+
+fun tensor(size0: Int, size1: Int, initializer: (Int, Int) -> Float) = initializedTensor(size0, size1) {
+    for (i0 in 0 until size0) {
+        for (i1 in 0 until size1) {
+            it[i0, i1] = initializer(i0, i1)
+        }
+    }
+}
+
+fun tensor(vararg values: Float) = tensor(values.size) { values[it] }
+fun tensor(vararg values: Array<Float>) = tensor(values.size, values.first().size) { i0, i1 -> values[i0][i1] }
+
+fun full(constant: Float, size: Int) = tensor(size) { constant }
+fun full(constant: Float, size0: Int, size1: Int) = tensor(size0, size1) { _, _ -> constant }
+fun full(constant: Float, shape: List<Int>) = when (shape.size) {
+    1 -> full(constant, shape.single())
+    2 -> full(constant, shape[0], shape[1])
+    else -> throw Error("Tensors with ${shape.size} dimensions are not supported yet.")
+}
+
+val randomGenerator = THGenerator_new()
+fun random(min: Float, max: Float) = THRandom_uniformFloat(randomGenerator, min, max)
+fun randomInt(count: Int, min: Int = 0) = random(min.toFloat(), count.toFloat()).toInt()
+fun random(min: Double, max: Double, size: Int) =
+        initializedTensor(size) { THFloatTensor_uniform(it.raw, randomGenerator, min, max) }
+
+fun random(min: Double, max: Double, size0: Int, size1: Int) =
+        initializedTensor(size0, size1) { THFloatTensor_uniform(it.raw, randomGenerator, min, max) }
+
+fun zeros(size: Int) = full(0f, size)
+fun zeros(size0: Int, size1: Int) = full(0f, size0, size1)
+fun zeros(shape: List<Int>) = full(0f, shape)
+
+fun ones(size: Int) = full(1f, size)
+fun ones(size0: Int, size1: Int) = full(1f, size0, size1)
+fun ones(shape: List<Int>) = full(1f, shape)