Move everything under kotlin-native folder
I was forced to manually do update the following files, because otherwise they would be ignored according .gitignore settings. Probably they should be deleted from repo. Interop/.idea/compiler.xml Interop/.idea/gradle.xml Interop/.idea/libraries/Gradle__org_jetbrains_kotlin_kotlin_runtime_1_0_3.xml Interop/.idea/libraries/Gradle__org_jetbrains_kotlin_kotlin_stdlib_1_0_3.xml Interop/.idea/modules.xml Interop/.idea/modules/Indexer/Indexer.iml Interop/.idea/modules/Runtime/Runtime.iml Interop/.idea/modules/StubGenerator/StubGenerator.iml backend.native/backend.native.iml backend.native/bc.frontend/bc.frontend.iml backend.native/cli.bc/cli.bc.iml backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt backend.native/tests/link/lib/foo.kt backend.native/tests/link/lib/foo2.kt backend.native/tests/teamcity-test.property
This commit is contained in:
Stanislav Erokhin
@@ -0,0 +1 @@
|
||||
headers = TH/TH.h THNN/THNN.h
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package sample.torch
|
||||
|
||||
fun Float.toRoundedString(digits: Int = 0): String {
|
||||
var factor = 1
|
||||
|
||||
for (i in 0 until digits) {
|
||||
factor *= 10
|
||||
}
|
||||
|
||||
return (kotlin.math.round(this * factor) / factor).toString()
|
||||
}
|
||||
|
||||
fun Float.toPercentageString(roundToDigits: Int = 1) = (this * 100).toRoundedString(roundToDigits)
|
||||
|
||||
fun List<Float>.maxIndex() = withIndex().maxBy { it.value }!!.index
|
||||
|
||||
fun accuracy(predictionBatch: FloatMatrix, labelBatch: FloatMatrix): Float {
|
||||
val resultIndexes = predictionBatch.toList().map { it.maxIndex() }
|
||||
val labelBatchIndexes = labelBatch.toList().map { it.maxIndex() }
|
||||
return resultIndexes.zip(labelBatchIndexes).
|
||||
count { (result, label) -> result == label }.toFloat() / resultIndexes.size
|
||||
}
|
||||
|
||||
fun Backpropagatable<FloatMatrix, FloatMatrix>.trainClassifier(
|
||||
dataset: Dataset,
|
||||
lossByLabels: (FloatMatrix) -> Backpropagatable<FloatMatrix, FloatVector>,
|
||||
learningRateByProgress: (Float) -> Float = { 5f * kotlin.math.exp(-it * 3) },
|
||||
batchSize: Int = 64,
|
||||
iterations: Int = 500) {
|
||||
|
||||
for (i in 0 until iterations) {
|
||||
disposeScoped {
|
||||
val (inputBatch, labelBatch) = dataset.sampleBatch(batchSize)
|
||||
val errorNetwork = this@trainClassifier before lossByLabels(labelBatch)
|
||||
val forwardResults = use { errorNetwork.forwardPass(inputBatch) }
|
||||
val accuracy = accuracy(forwardResults.hidden, labelBatch)
|
||||
val progress = i.toFloat() / iterations
|
||||
val learningRate = learningRateByProgress(progress)
|
||||
val backpropResults = use { forwardResults.backpropagate(outputGradient = tensor(learningRate)) }
|
||||
val crossEntropy = forwardResults.output[0]
|
||||
backpropResults.descend()
|
||||
println("Iteration ${i + 1}/$iterations: " +
|
||||
"${accuracy.toPercentageString()}% training batch accuracy, " +
|
||||
"cross entropy loss = ${crossEntropy.toRoundedString(4)}, " +
|
||||
"learning rate = ${learningRate.toRoundedString(4)}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun Backpropagatable<FloatMatrix, FloatMatrix>.testClassifier(dataset: Dataset, batchSize: Int = 100): Float {
|
||||
val testBatches = dataset.testBatches(batchSize)
|
||||
return testBatches.withIndex().map { (i, batchPair) ->
|
||||
val (inputBatch, outputBatch) = batchPair
|
||||
val accuracy = accuracy(this.forwardPass(inputBatch).output, outputBatch)
|
||||
println("Test batch ${i + 1}/${testBatches.size}: ${accuracy.toPercentageString()}% accuracy")
|
||||
accuracy * inputBatch.shape[0]
|
||||
}.sum() / dataset.inputs.size
|
||||
}
|
||||
|
||||
fun randomInit(size: Int) = random(-.01, .01, size)
|
||||
fun randomInit(size0: Int, size1: Int) = random(-.1, .1, size0, size1)
|
||||
|
||||
fun linear(inputSize: Int, outputSize: Int) = Linear(randomInit(outputSize, inputSize), randomInit(outputSize))
|
||||
fun twoLayerClassifier(dataset: Dataset, hiddenSize: Int = 64) =
|
||||
linear(dataset.inputs[0].size, hiddenSize) before Relu before
|
||||
linear(hiddenSize, dataset.labels[0].size) before Softmax
|
||||
|
||||
fun main() {
|
||||
val trainingDataset = MNIST.labeledTrainingImages()
|
||||
val predictionNetwork = twoLayerClassifier(trainingDataset)
|
||||
predictionNetwork.trainClassifier(trainingDataset, lossByLabels = { CrossEntropyLoss(labels = it) })
|
||||
|
||||
val testDataset = MNIST.labeledTestImages()
|
||||
val averageAccuracy = predictionNetwork.testClassifier(testDataset)
|
||||
println("Accuracy on the test set: ${averageAccuracy.toPercentageString()}")
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package sample.torch
|
||||
|
||||
import kotlinx.cinterop.*
|
||||
import platform.posix.*
|
||||
|
||||
data class Dataset(val inputs: List<FloatArray>, val labels: List<FloatArray>) {
|
||||
fun batch(indices: List<Int>): Pair<FloatMatrix, FloatMatrix> {
|
||||
val inputBatch = tensor(*(indices.map { inputs[it].toTypedArray() }.toTypedArray()))
|
||||
val labelBatch = tensor(*(indices.map { labels[it].toTypedArray() }.toTypedArray()))
|
||||
return inputBatch to labelBatch
|
||||
}
|
||||
|
||||
fun sampleBatch(batchSize: Int) = batch((0 until batchSize).map { randomInt(inputs.size) })
|
||||
private fun batchAt(batchIndex: Int, batchSize: Int) =
|
||||
batch((0 until inputs.size).drop(batchSize + batchIndex).take(batchSize))
|
||||
|
||||
fun testBatches(batchSize: Int) = (0 until inputs.size / batchSize).map { batchAt(it, batchSize = batchSize) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides the MNIST labeled handwritten digit dataset, described at http://yann.lecun.com/exdb/mnist/
|
||||
*/
|
||||
object MNIST {
|
||||
private fun readFileData(fileName: String) = memScoped {
|
||||
val path = "build/3rd-party/MNIST/$fileName"
|
||||
fun fail(): Nothing = throw Error("Cannot read input file $path")
|
||||
|
||||
val size = alloc<stat>().also { if (stat(path, it.ptr) != 0) fail() }.st_size.toInt()
|
||||
|
||||
println("Reading $size bytes from $path...")
|
||||
|
||||
val file = fopen(path, "rb") ?: fail()
|
||||
try {
|
||||
ByteArray(size).also { fread(it.refTo(0), 1, size.convert(), file) }
|
||||
} finally {
|
||||
fclose(file)
|
||||
}
|
||||
}
|
||||
|
||||
private fun Byte.reinterpretAsUnsigned() = this.toInt().let { it + if (it < 0) 256 else 0 }
|
||||
|
||||
private fun unsignedBytesToInt(bytes: List<Byte>) =
|
||||
bytes.withIndex().map { (i, value) -> value.reinterpretAsUnsigned().shl(8 * (3 - i)) }.sum()
|
||||
|
||||
private val intSize = 4
|
||||
private fun ByteArray.getIntAt(index: Int) =
|
||||
unsignedBytesToInt((index until (index + intSize)).map { this[it] })
|
||||
|
||||
private val imageLength = 28
|
||||
private val imageSize = imageLength * imageLength
|
||||
|
||||
private fun ByteArray.getImageAt(index: Int) =
|
||||
FloatArray(imageSize) { this[index + it].reinterpretAsUnsigned().toFloat() / 255 }
|
||||
|
||||
private fun oneHot(size: Int, index: Int) = FloatArray(size) { if (it == index) 1f else 0f }
|
||||
|
||||
private fun readLabels(fileName: String, totalLabels: Int = 10): List<FloatArray> {
|
||||
val data = readFileData(fileName)
|
||||
val check = data.getIntAt(0)
|
||||
val expectedCheck = 2049
|
||||
if (check != 2049) throw Error("File should start with int $expectedCheck, but was $check.")
|
||||
|
||||
val count = data.getIntAt(4)
|
||||
|
||||
val offset = 8
|
||||
|
||||
if (count + offset != data.size) throw Error("Unexpected file size: ${data.size}.")
|
||||
|
||||
return (0 until count).map { oneHot(totalLabels, index = data[offset + it].reinterpretAsUnsigned()) }
|
||||
}
|
||||
|
||||
private fun readImages(fileName: String): List<FloatArray> {
|
||||
val data = readFileData(fileName)
|
||||
val check = data.getIntAt(0)
|
||||
val expectedCheck = 2051
|
||||
if (check != expectedCheck) throw Error("File should start with int $expectedCheck, but was $check.")
|
||||
|
||||
val count = data.getIntAt(4)
|
||||
val width = data.getIntAt(8)
|
||||
val height = data.getIntAt(12)
|
||||
|
||||
val offset = 16
|
||||
|
||||
if (width != imageLength) throw Error()
|
||||
if (height != imageLength) throw Error()
|
||||
|
||||
if (count * imageSize + offset != data.size) throw Error("Unexpected file size: ${data.size}.")
|
||||
|
||||
return (0 until count).map { data.getImageAt(offset + imageSize * it) }
|
||||
}
|
||||
|
||||
fun labeledTrainingImages() = Dataset(
|
||||
inputs = readImages("train-images-idx3-ubyte"),
|
||||
labels = readLabels("train-labels-idx1-ubyte"))
|
||||
|
||||
fun labeledTestImages() = Dataset(
|
||||
inputs = readImages("t10k-images-idx3-ubyte"),
|
||||
labels = readLabels("t10k-labels-idx1-ubyte"))
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package sample.torch
|
||||
|
||||
/**
|
||||
* NOTE: resource management in this sample suffers from resource leaks
|
||||
* and double-free bugs (see workaround in [FloatTensor.dispose]).
|
||||
*
|
||||
* This might mean that the entire approach for resource management in the sample is faulty.
|
||||
* Please take this into account when considering reusing the same approach in your project.
|
||||
*
|
||||
* TODO: rework resource management.
|
||||
*/
|
||||
interface Disposable {
|
||||
fun dispose()
|
||||
}
|
||||
|
||||
open class DisposableContainer(private val disposables: MutableList<Disposable> = ArrayList()) : Disposable {
|
||||
/**
|
||||
* Creates the object and schedules its disposal for the end of the scope.
|
||||
*/
|
||||
fun <T : Disposable> use(create: () -> T) = create().also { disposables.add(it) }
|
||||
|
||||
override fun dispose() {
|
||||
for (disposable in disposables) {
|
||||
disposable.dispose()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun <T> disposeScoped(action: DisposableContainer.() -> T): T {
|
||||
val scope = DisposableContainer()
|
||||
|
||||
try {
|
||||
return scope.action()
|
||||
} finally {
|
||||
scope.dispose()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
/*
|
||||
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package sample.torch
|
||||
|
||||
import kotlinx.cinterop.*
|
||||
import torch.*
|
||||
|
||||
// Defines network modules with the ability for backpropagation using both TH.h and THNN.h from the ATen library
|
||||
|
||||
abstract class Backpropagatable<Input : Disposable, Output : Disposable> {
|
||||
abstract inner class ForwardResults(val input: Input) : DisposableContainer() {
|
||||
init {
|
||||
use { input }
|
||||
}
|
||||
|
||||
abstract val output: Output
|
||||
abstract fun backpropagate(outputGradient: Output): BackpropagationResults
|
||||
}
|
||||
|
||||
abstract inner class BackpropagationResults(
|
||||
val input: Input,
|
||||
val output: Output,
|
||||
val outputGradient: Output
|
||||
) : DisposableContainer() {
|
||||
init {
|
||||
use { input }
|
||||
use { output }
|
||||
use { outputGradient }
|
||||
}
|
||||
|
||||
abstract val inputGradient: Input
|
||||
abstract fun descend()
|
||||
}
|
||||
|
||||
abstract fun forwardPass(input: Input): ForwardResults
|
||||
}
|
||||
|
||||
abstract class Module<Input : Disposable, Output : Disposable, Parameters> : Backpropagatable<Input, Output>() {
|
||||
abstract var parameters: Parameters
|
||||
abstract fun parametersToList(parameters: Parameters): List<FloatTensor>
|
||||
abstract fun parametersFromList(list: List<FloatTensor>): Parameters
|
||||
private val parameterList get() = parametersToList(parameters)
|
||||
|
||||
abstract operator fun invoke(input: Input): Output
|
||||
abstract fun inputGradient(input: Input, outputGradient: Output, output: Output): Input
|
||||
abstract fun parameterGradient(input: Input, outputGradient: Output, inputGradient: Input): Parameters
|
||||
|
||||
override fun forwardPass(input: Input) = object : ForwardResults(input) {
|
||||
override val output = use { this@Module(input) }
|
||||
override fun backpropagate(outputGradient: Output) =
|
||||
object : Backpropagatable<Input, Output>.BackpropagationResults(input, output, outputGradient) {
|
||||
override val inputGradient = use { this@Module.inputGradient(input, outputGradient, output) }
|
||||
val parameterGradient = this@Module.parameterGradient(input,
|
||||
outputGradient = outputGradient, inputGradient = inputGradient)
|
||||
|
||||
override fun descend() = this@Module.descend(parameterGradient)
|
||||
}
|
||||
}
|
||||
|
||||
open fun descend(parameterGradient: Parameters) {
|
||||
parameters = parametersFromList(parameterList.zip(
|
||||
parametersToList(parameterGradient)) { parameter, gradient -> parameter - gradient })
|
||||
}
|
||||
}
|
||||
|
||||
abstract class ParameterFreeModule<Input : Disposable, Output : Disposable> : Module<Input, Output, Unit>() {
|
||||
override var parameters = Unit
|
||||
override fun parametersToList(parameters: Unit) = emptyList<FloatTensor>()
|
||||
override fun parametersFromList(list: List<FloatTensor>) = Unit
|
||||
override fun parameterGradient(input: Input, outputGradient: Output, inputGradient: Input) = Unit
|
||||
override fun descend(parameterGradient: Unit) {}
|
||||
}
|
||||
|
||||
class Chain<Input : Disposable, Hidden : Disposable, Output : Disposable>(
|
||||
val module1: Backpropagatable<Input, Hidden>,
|
||||
val module2: Backpropagatable<Hidden, Output>
|
||||
) : Backpropagatable<Input, Output>() {
|
||||
override fun forwardPass(input: Input) = ChainForwardResults(input)
|
||||
|
||||
inner class ChainForwardResults(input: Input) : ForwardResults(input) {
|
||||
val result1 = use { module1.forwardPass(input) }
|
||||
val hidden = result1.output
|
||||
val result2 = use { module2.forwardPass(result1.output) }
|
||||
override val output = result2.output
|
||||
override fun backpropagate(outputGradient: Output) =
|
||||
object : Backpropagatable<Input, Output>.BackpropagationResults(input, output, outputGradient) {
|
||||
val backpropResults2 = use { result2.backpropagate(outputGradient) }
|
||||
val hiddenGradient = backpropResults2.inputGradient
|
||||
val backpropResults1 = use { result1.backpropagate(hiddenGradient) }
|
||||
|
||||
override val inputGradient = backpropResults1.inputGradient
|
||||
|
||||
override fun descend() {
|
||||
backpropResults1.descend()
|
||||
backpropResults2.descend()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun toString() = "$module1 before $module2"
|
||||
}
|
||||
|
||||
infix fun <Input : Disposable, Hidden : Disposable, Output : Disposable> Backpropagatable<Input, Hidden>.before(
|
||||
other: Backpropagatable<Hidden, Output>) = Chain(this, other)
|
||||
|
||||
object Abs : ParameterFreeModule<FloatMatrix, FloatMatrix>() {
|
||||
override operator fun invoke(input: FloatMatrix) = initializedTensor(input.shape[0], input.shape[1]) {
|
||||
THNN_FloatAbs_updateOutput(cValuesOf<FloatVar>(), input.raw, it.raw)
|
||||
}
|
||||
|
||||
override fun inputGradient(input: FloatMatrix, outputGradient: FloatMatrix, output: FloatMatrix) =
|
||||
initializedTensor(input.shape[0], input.shape[1]) {
|
||||
THNN_FloatAbs_updateGradInput(null, input.raw, outputGradient.raw, it.raw)
|
||||
}
|
||||
}
|
||||
|
||||
object Relu : ParameterFreeModule<FloatMatrix, FloatMatrix>() {
|
||||
override operator fun invoke(input: FloatMatrix) = initializedTensor(input.shape[0], input.shape[1]) {
|
||||
THNN_FloatLeakyReLU_updateOutput(null, input.raw, it.raw, 0.0, false)
|
||||
}
|
||||
|
||||
override fun inputGradient(input: FloatMatrix, outputGradient: FloatMatrix, output: FloatMatrix) =
|
||||
initializedTensor(input.shape[0], input.shape[1]) {
|
||||
THNN_FloatLeakyReLU_updateGradInput(null, input.raw, outputGradient.raw, it.raw, 0.0, false)
|
||||
}
|
||||
}
|
||||
|
||||
object Softmax : ParameterFreeModule<FloatMatrix, FloatMatrix>() {
|
||||
override operator fun invoke(input: FloatMatrix) = initializedTensor(input.shape[0], input.shape[1]) {
|
||||
THNN_FloatSoftMax_updateOutput(null, input.raw, it.raw, 1)
|
||||
}
|
||||
|
||||
override fun inputGradient(input: FloatMatrix, outputGradient: FloatMatrix, output: FloatMatrix) =
|
||||
initializedTensor(input.shape[0], input.shape[1]) {
|
||||
THNN_FloatSoftMax_updateGradInput(null, input.raw, outputGradient.raw, it.raw, output.raw, 1)
|
||||
}
|
||||
}
|
||||
|
||||
class MeanSquaredError(val labels: FloatMatrix) : ParameterFreeModule<FloatMatrix, FloatVector>() {
|
||||
override operator fun invoke(input: FloatMatrix) = initializedTensor(1) {
|
||||
THNN_FloatMSECriterion_updateOutput(null, input.raw, labels.raw, it.raw,
|
||||
sizeAverage = true, reduce = true)
|
||||
}
|
||||
|
||||
override fun inputGradient(
|
||||
input: FloatMatrix,
|
||||
outputGradient: FloatVector,
|
||||
output: FloatVector
|
||||
) = initializedTensor(input.shape[0], input.shape[1]) {
|
||||
THNN_FloatMSECriterion_updateGradInput(null, input.raw, labels.raw,
|
||||
outputGradient.raw, it.raw, sizeAverage = true, reduce = true)
|
||||
}
|
||||
}
|
||||
|
||||
class CrossEntropyLoss(val labels: FloatMatrix) : ParameterFreeModule<FloatMatrix, FloatVector>() {
|
||||
override operator fun invoke(input: FloatMatrix) = initializedTensor(1) {
|
||||
THNN_FloatBCECriterion_updateOutput(null, input.raw, labels.raw, it.raw,
|
||||
sizeAverage = true, reduce = true, weights = null)
|
||||
}
|
||||
|
||||
override fun inputGradient(input: FloatMatrix, outputGradient: FloatVector, output: FloatVector) =
|
||||
initializedTensor(input.shape[0], input.shape[1]) {
|
||||
THNN_FloatBCECriterion_updateGradInput(null, input.raw, labels.raw, outputGradient.raw,
|
||||
it.raw, sizeAverage = true, reduce = true, weights = null)
|
||||
}
|
||||
}
|
||||
|
||||
data class Linear(
|
||||
var weight: FloatMatrix,
|
||||
var bias: FloatVector) : Module<FloatMatrix, FloatMatrix, Pair<FloatMatrix, FloatVector>>() {
|
||||
val inputSize = weight.shape[1]
|
||||
val outputSize = weight.shape[0]
|
||||
val addBuffer = uninitializedTensor(outputSize)
|
||||
|
||||
override operator fun invoke(input: FloatMatrix) = initializedTensor(input.shape[0], outputSize) {
|
||||
THNN_FloatLinear_updateOutput(null, input.raw, it.raw, weight.raw, bias.raw, addBuffer.raw)
|
||||
}
|
||||
|
||||
override fun inputGradient(input: FloatMatrix, outputGradient: FloatMatrix, output: FloatMatrix) =
|
||||
initializedTensor(input.shape[0], inputSize) {
|
||||
THNN_FloatLinear_updateGradInput(null, input.raw, outputGradient.raw, it.raw, weight.raw)
|
||||
}
|
||||
|
||||
override fun parameterGradient(
|
||||
input: FloatMatrix,
|
||||
outputGradient: FloatMatrix,
|
||||
inputGradient: FloatMatrix
|
||||
): Pair<FloatMatrix, FloatVector> {
|
||||
val biasGradient = zeros(outputSize)
|
||||
val weightGradient = zeros(weight.shape[0], weight.shape[1]).also {
|
||||
THNN_FloatLinear_accGradParameters(null, input.raw, outputGradient.raw, inputGradient.raw, weight.raw,
|
||||
bias.raw, it.raw, biasGradient.raw, addBuffer.raw, 1.0)
|
||||
}
|
||||
|
||||
return weightGradient to biasGradient
|
||||
}
|
||||
|
||||
override var parameters: Pair<FloatMatrix, FloatVector>
|
||||
get() = weight to bias
|
||||
set(value) {
|
||||
weight = value.first
|
||||
bias = value.second
|
||||
}
|
||||
|
||||
override fun parametersToList(parameters: Pair<FloatMatrix, FloatVector>) =
|
||||
listOf(parameters.first, parameters.second)
|
||||
|
||||
override fun parametersFromList(list: List<FloatTensor>) = list.first().asMatrix() to list.last().asVector()
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package sample.torch
|
||||
|
||||
// If you are curious you can also try one of these
|
||||
|
||||
private fun demonstrateTensors() {
|
||||
disposeScoped {
|
||||
val x = use { tensor(0f, 1f, 2f) }
|
||||
val y = use { tensor(0f, -1f, -2f) }
|
||||
val m = use {
|
||||
tensor(
|
||||
arrayOf(1f, -1f, 0f),
|
||||
arrayOf(0f, -1f, 0f),
|
||||
arrayOf(0f, 0f, -.5f))
|
||||
}
|
||||
|
||||
println("Hello, Torch!\nx = $x\ny = $y\n" +
|
||||
"|x| = ${x.abs()}\n|y| = ${y.abs()}\n" +
|
||||
"2x=${use { x * 2f }}\nx+y = ${use { x + y }}\nx-y = ${use { x - y }}\nxy = ${x * y}\n" +
|
||||
"m=\n${use { m }}\nm·y = ${use { m * y }}\nm+m =\n${use { m + m }}\nm·m =\n${use { m * m }}")
|
||||
}
|
||||
}
|
||||
|
||||
private fun demonstrateModules() {
|
||||
val input = tensor(arrayOf(-1f))
|
||||
val abs = Abs(input)
|
||||
println("abs of $input is $abs, gradient is ${Abs.inputGradient(input, tensor(arrayOf(1f)), abs)}")
|
||||
val relu = Relu(input)
|
||||
println("relu of $input is $relu, gradient is ${Relu.inputGradient(input, tensor(arrayOf(1f)), relu)}")
|
||||
}
|
||||
|
||||
private fun demonstrateManualBackpropagationFor1LinearLayer(
|
||||
inputs: FloatMatrix = tensor(arrayOf(1f, -1f), arrayOf(1f, -1f)),
|
||||
labels: FloatMatrix = tensor(arrayOf(5f), arrayOf(5f)),
|
||||
learningRate: Float = .1f) {
|
||||
val linear = Linear(weight = randomInit(1, 2), bias = randomInit(1))
|
||||
val error = MeanSquaredError(labels)
|
||||
|
||||
for (i in 0 until 100) {
|
||||
disposeScoped {
|
||||
val output = use { linear(inputs) }
|
||||
val loss = use { error(output) }
|
||||
val outputGradient = use { error.inputGradient(output, tensor(learningRate), loss) }
|
||||
val inputGradient = use { linear.inputGradient(inputs, outputGradient, output) }
|
||||
val parameterGradient = linear.parameterGradient(inputs, outputGradient, inputGradient).
|
||||
also { use { it.first } }.also { use { it.second } }
|
||||
println("input: $inputs, \n" +
|
||||
"output: $output, \n" +
|
||||
"labels: $labels, \n" +
|
||||
"mean squared error: $loss, \n" +
|
||||
"output gradient: $outputGradient, \n" +
|
||||
"input gradient: $inputGradient, \n" +
|
||||
"parameter gradient: $parameterGradient")
|
||||
linear.weight -= parameterGradient.first
|
||||
linear.bias -= parameterGradient.second
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
/*
|
||||
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
* that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package sample.torch
|
||||
|
||||
import kotlinx.cinterop.*
|
||||
import torch.*
|
||||
|
||||
// Defines tensor classes and operations using TH.h from the ATen library
|
||||
|
||||
abstract class FloatTensor(val raw: CPointer<THFloatTensor>) : Disposable {
|
||||
private val storage: CPointer<THFloatStorage> get() = raw.pointed.storage!!
|
||||
private val elements get() = storage.pointed
|
||||
private val data: CPointer<FloatVar> get() = elements.data!!
|
||||
private val size: CPointer<LongVar> get() = raw.pointed.size!!
|
||||
protected val nDimension: Int get() = raw.pointed.nDimension
|
||||
|
||||
val shape: List<Int> get() = (0 until nDimension).map { size[it].toInt() }
|
||||
|
||||
operator fun plus(other: FloatTensor) = initializedTensor(shape) {
|
||||
THFloatTensor_cadd(it.raw, raw, 1f, other.raw)
|
||||
}
|
||||
|
||||
operator fun minus(other: FloatTensor) = initializedTensor(shape) {
|
||||
THFloatTensor_cadd(it.raw, raw, -1f, other.raw)
|
||||
}
|
||||
|
||||
open operator fun times(factor: Float) = initializedTensor(shape) {
|
||||
THFloatTensor_mul(it.raw, raw, factor)
|
||||
}
|
||||
|
||||
fun sum() = THFloatTensor_sumall(raw)
|
||||
fun flatten() = (0 until elements.size).map { data[it] }.toTypedArray()
|
||||
|
||||
private var disposed = false
|
||||
override fun dispose() {
|
||||
if (disposed) return
|
||||
disposed = true
|
||||
|
||||
THFloatTensor_free(raw)
|
||||
}
|
||||
|
||||
fun asVector() = FloatVector(raw)
|
||||
fun asMatrix() = FloatMatrix(raw)
|
||||
inline fun <reified T : FloatTensor> asTensor(): T = when (T::class) {
|
||||
FloatVector::class -> asVector() as T
|
||||
FloatMatrix::class -> asMatrix() as T
|
||||
FloatTensor::class -> this as T
|
||||
else -> throw Error("Unexpected class ${T::class}")
|
||||
}
|
||||
|
||||
abstract override fun toString(): String
|
||||
}
|
||||
|
||||
class FloatVector(raw: CPointer<THFloatTensor>) : FloatTensor(raw) {
|
||||
init {
|
||||
if (super.nDimension != 1)
|
||||
throw Error("A vector must have exactly 1 dimension.")
|
||||
}
|
||||
|
||||
operator fun get(i: Int) = THFloatTensor_get1d(raw, i.signExtend())
|
||||
operator fun set(i: Int, value: Float) = THFloatTensor_set1d(raw, i.signExtend(), value)
|
||||
fun toArray() = (0 until shape[0]).map { i0 -> this[i0] }.toTypedArray()
|
||||
|
||||
operator fun plus(other: FloatVector) = super.plus(other).asVector()
|
||||
operator fun minus(other: FloatVector) = super.minus(other).asVector()
|
||||
override operator fun times(factor: Float) = super.times(factor).asVector()
|
||||
operator fun times(other: FloatVector) = THFloatTensor_dot(raw, other.raw)
|
||||
|
||||
fun abs() = kotlin.math.sqrt(this * this)
|
||||
|
||||
override fun toString() = "[${toArray().joinToString { it.toString() }}]"
|
||||
}
|
||||
|
||||
class FloatMatrix(raw: CPointer<THFloatTensor>) : FloatTensor(raw) {
|
||||
init {
|
||||
if (super.nDimension != 2)
|
||||
throw Error("A matrix must have exactly 2 dimensions.")
|
||||
}
|
||||
|
||||
fun getRow(i0: Int) = (0 until shape[1]).map { i1 -> this[i0, i1] }
|
||||
operator fun get(i0: Int, i1: Int) = THFloatTensor_get2d(raw, i0.signExtend(), i1.signExtend())
|
||||
operator fun set(i0: Int, i1: Int, value: Float) = THFloatTensor_set2d(raw, i0.signExtend(), i1.signExtend(), value)
|
||||
fun toList() = (0 until shape[0]).map { getRow(it) }
|
||||
|
||||
operator fun plus(other: FloatMatrix) = super.plus(other).asMatrix()
|
||||
operator fun minus(other: FloatMatrix) = super.minus(other).asMatrix()
|
||||
override operator fun times(factor: Float) = super.times(factor).asMatrix()
|
||||
|
||||
operator fun times(vector: FloatVector) = initializedTensor(shape[0]) {
|
||||
THFloatTensor_addmv(it.raw, 0f, it.raw, 1f, raw, vector.raw)
|
||||
}
|
||||
|
||||
operator fun times(matrix: FloatMatrix) = initializedTensor(shape[0], matrix.shape[1]) {
|
||||
THFloatTensor_addmm(it.raw, 0f, it.raw, 1f, raw, matrix.raw)
|
||||
}
|
||||
|
||||
override fun toString() = "[${toList().joinToString(",\n") { "[${it.joinToString { it.toString() }}]" }}]"
|
||||
}
|
||||
|
||||
fun uninitializedTensor(size: Int) =
|
||||
FloatVector(THFloatTensor_newWithSize1d(size.signExtend())!!)
|
||||
|
||||
fun uninitializedTensor(size0: Int, size1: Int) =
|
||||
FloatMatrix(THFloatTensor_newWithSize2d(size0.signExtend(), size1.signExtend())!!)
|
||||
|
||||
fun uninitializedTensor(shape: List<Int>) = when (shape.size) {
|
||||
1 -> uninitializedTensor(shape.single())
|
||||
2 -> uninitializedTensor(shape[0], shape[1])
|
||||
else -> throw Error("Tensors with ${shape.size} dimensions are not supported yet.")
|
||||
}
|
||||
|
||||
fun <T> initializedTensor(size: Int, initializer: (FloatVector) -> T) =
|
||||
uninitializedTensor(size).apply { initializer(this) }
|
||||
|
||||
fun <T> initializedTensor(size0: Int, size1: Int, initializer: (FloatMatrix) -> T) =
|
||||
uninitializedTensor(size0, size1).apply { initializer(this) }
|
||||
|
||||
fun <T> initializedTensor(shape: List<Int>, initializer: (FloatTensor) -> T) =
|
||||
uninitializedTensor(shape).apply { initializer(this) }
|
||||
|
||||
fun tensor(size: Int, initializer: (Int) -> Float) = initializedTensor(size) {
|
||||
for (i in 0 until size) {
|
||||
it[i] = initializer(i)
|
||||
}
|
||||
}
|
||||
|
||||
fun tensor(size0: Int, size1: Int, initializer: (Int, Int) -> Float) = initializedTensor(size0, size1) {
|
||||
for (i0 in 0 until size0) {
|
||||
for (i1 in 0 until size1) {
|
||||
it[i0, i1] = initializer(i0, i1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun tensor(vararg values: Float) = tensor(values.size) { values[it] }
|
||||
fun tensor(vararg values: Array<Float>) = tensor(values.size, values.first().size) { i0, i1 -> values[i0][i1] }
|
||||
|
||||
fun full(constant: Float, size: Int) = tensor(size) { constant }
|
||||
fun full(constant: Float, size0: Int, size1: Int) = tensor(size0, size1) { _, _ -> constant }
|
||||
fun full(constant: Float, shape: List<Int>) = when (shape.size) {
|
||||
1 -> full(constant, shape.single())
|
||||
2 -> full(constant, shape[0], shape[1])
|
||||
else -> throw Error("Tensors with ${shape.size} dimensions are not supported yet.")
|
||||
}
|
||||
|
||||
val randomGenerator = THGenerator_new()
|
||||
fun random(min: Float, max: Float) = THRandom_uniformFloat(randomGenerator, min, max)
|
||||
fun randomInt(count: Int, min: Int = 0) = random(min.toFloat(), count.toFloat()).toInt()
|
||||
fun random(min: Double, max: Double, size: Int) =
|
||||
initializedTensor(size) { THFloatTensor_uniform(it.raw, randomGenerator, min, max) }
|
||||
|
||||
fun random(min: Double, max: Double, size0: Int, size1: Int) =
|
||||
initializedTensor(size0, size1) { THFloatTensor_uniform(it.raw, randomGenerator, min, max) }
|
||||
|
||||
fun zeros(size: Int) = full(0f, size)
|
||||
fun zeros(size0: Int, size1: Int) = full(0f, size0, size1)
|
||||
fun zeros(shape: List<Int>) = full(0f, shape)
|
||||
|
||||
fun ones(size: Int) = full(1f, size)
|
||||
fun ones(size0: Int, size1: Int) = full(1f, size0, size1)
|
||||
fun ones(shape: List<Int>) = full(1f, shape)
|
||||
Reference in New Issue
Block a user