Python extension sample. (#1123)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt

@@ -208,7 +208,7 @@ private class ExportedElement(val kind: ElementKind,
 
     fun makeClassDeclaration(): String {
         assert(isClass)
-        return "extern \"C\" ${owner.prefix}_KType* $cname();"
+        return "extern \"C\" ${owner.prefix}_KType* $cname(void);"
     }
 
     private fun translateArgument(name: String, clazz: ClassDescriptor, direction: Direction,
@@ -366,7 +366,7 @@ internal class CAdapterGenerator(val context: Context,
                     element.isFunction ->
                         output(element.makeFunctionPointerString(), indent)
                     element.isClass ->
-                        output("${prefix}_KType* (*_type)();", indent)
+                        output("${prefix}_KType* (*_type)(void);", indent)
                 // TODO: handle properties.
                 }
             }
@@ -455,7 +455,7 @@ internal class CAdapterGenerator(val context: Context,
         output("typedef struct {")
         output("/* Service functions. */", 1)
         output("void (*DisposeStablePointer)(${prefix}_KNativePtr ptr);", 1)
-        output("void (*DisposeString)(char* string);", 1)
+        output("void (*DisposeString)(const char* string);", 1)
         output("${prefix}_KBoolean (*IsInstance)(${prefix}_KNativePtr ref, const ${prefix}_KType* type);", 1)
 
         output("")
@@ -463,7 +463,7 @@ internal class CAdapterGenerator(val context: Context,
         makeScopeDefinitions(top, DefinitionKind.C_HEADER_STRUCT, 1)
         output("} ${prefix}_ExportedSymbols;")
 
-        output("extern ${prefix}_ExportedSymbols* ${prefix}_symbols();")
+        output("extern ${prefix}_ExportedSymbols* ${prefix}_symbols(void);")
         output("""
         #ifdef __cplusplus
         }  /* extern "C" */
@@ -490,7 +490,7 @@ internal class CAdapterGenerator(val context: Context,
         |#define RUNTIME_NOTHROW __attribute__((nothrow))
         |#define RUNTIME_USED __attribute__((used))
         |
-        |void SetRef(KObjHeader**, const KObjHeader*) RUNTIME_NOTHROW;
+        |extern "C" {
         |void UpdateRef(KObjHeader**, const KObjHeader*) RUNTIME_NOTHROW;
         |KObjHeader* AllocInstance(const KTypeInfo*, KObjHeader**) RUNTIME_NOTHROW;
         |KObjHeader* DerefStablePointer(void*, KObjHeader**) RUNTIME_NOTHROW;
@@ -502,12 +502,13 @@ internal class CAdapterGenerator(val context: Context,
         |KObjHeader* CreateStringFromCString(const char*, KObjHeader**);
         |char* CreateCStringFromString(const KObjHeader*);
         |void DisposeCString(char* cstring);
+        |}  // extern "C"
         |
         |class KObjHolder {
         |public:
         |  KObjHolder() : obj_(nullptr) {}
-        |  explicit KObjHolder(const KObjHeader* obj) {
-        |    SetRef(&obj_, obj);
+        |  explicit KObjHolder(const KObjHeader* obj) : obj_(nullptr) {
+        |    UpdateRef(&obj_, obj);
         |  }
         |  ~KObjHolder() {
         |    UpdateRef(&obj_, nullptr);
@@ -520,8 +521,8 @@ internal class CAdapterGenerator(val context: Context,
         |static void DisposeStablePointerImpl(${prefix}_KNativePtr ptr) {
         |  DisposeStablePointer(ptr);
         |}
-        |static void DisposeStringImpl(char* ptr) {
-        |  DisposeCString(ptr);
+        |static void DisposeStringImpl(const char* ptr) {
+        |  DisposeCString((char*)ptr);
         |}
         |static ${prefix}_KBoolean IsInstanceImpl(${prefix}_KNativePtr ref, const ${prefix}_KType* type) {
         |  KObjHolder holder;
@@ -535,7 +536,7 @@ internal class CAdapterGenerator(val context: Context,
         output(".IsInstance = IsInstanceImpl,", 1)
         makeScopeDefinitions(top, DefinitionKind.C_SOURCE_STRUCT, 1)
         output("};")
-        output("RUNTIME_USED ${prefix}_ExportedSymbols* ${prefix}_symbols() { return &__konan_symbols;}")
+        output("RUNTIME_USED ${prefix}_ExportedSymbols* ${prefix}_symbols(void) { return &__konan_symbols;}")
         outputStreamWriter.close()
     }
 

samples/python_extension/README.md

@@ -0,0 +1,99 @@
+# C to Kotlin/Native interoperability example
+
+This example shows how to use Kotlin/Native programs from other execution environments, such as Python.
+Python has C native interface, which could be used to organize a bridge with the
+Kotlin/Native application. File `kotlin_bridge.c` contains translation code between Kotlin and Python
+lands.
+
+To build and run the sample do the following:
+
+*   Run `build.sh`, it will ask for superuser password to install Python extension
+*   On macOS copy Kotlin binary to extension's directory and change install name with
+    `install_name_tool` tool, i.e.
+    ```
+    sudo cp libserver.dylib  /Library/Python/2.7/site-packages/
+    sudo install_name_tool /Library/Python/2.7/site-packages/kotlin_bridge.so \
+          -change libserver.dylib @loader_path/libserver.dylib
+    ```
+
+*   run Python code using Kotlin functionality with
+    ```
+    python src/main/python/main.py
+    ```
+*   it will show you result of using several Kotlin/Native APIs, accepting and returning both objects and
+    primitive types
+
+ The example works as following. Kotlin/Native API is implemented in `Server.kt`, and we run Kotlin/Native compiler
+ with `-produce dynamic` option. Compiler produces two artifacts: `server_api.h` which is C language API
+ to all public functions and classes available in the application. `libserver.dylib` or `libserver.so`
+ shared object contains C bridge to all above APIs.
+
+  This C bridge looks like a C struct, reflecting all scopes in program, with operations available. For example,
+  for class Server
+```c_cpp
+   class Server(val prefix: String) {
+          fun greet(session: Session) = "$prefix: Hello from Kotlin/Native in ${session}"
+          fun concat(session: Session, a: String, b: String) = "$prefix: $a $b in ${session}"
+          fun add(session: Session, a: Int, b: Int) = a + b + session.number
+   }
+```
+   following C API is produced
+```c_cpp
+   typedef struct {
+     server_KNativePtr pinned;
+   } server_kref_demo_Session;
+   typedef struct {
+     server_KNativePtr pinned;
+   } server_kref_demo_Server;
+
+   typedef struct {
+      /* Service functions. */
+      void (*DisposeStablePointer)(server_KNativePtr ptr);
+      void (*DisposeString)(const char* string);
+      server_KBoolean (*IsInstance)(server_KNativePtr ref, const server_KType* type);
+
+      /* User functions. */
+      struct {
+        struct {
+          struct {
+            server_KType* (*_type)(void);
+            server_kref_demo_Session (*Session)(const char* name, server_KInt number);
+          } Session;
+          struct {
+            server_KType* (*_type)(void);
+            server_kref_demo_Server (*Server)(const char* prefix);
+            const char* (*greet)(server_kref_demo_Server thiz, server_kref_demo_Session session);
+            const char* (*concat)(server_kref_demo_Server thiz, server_kref_demo_Session session, const char* a, const char* b);
+            server_KInt (*add)(server_kref_demo_Server thiz, server_kref_demo_Session session, server_KInt a, server_KInt b);
+          } Server;
+        } demo;
+      } kotlin;
+   } server_ExportedSymbols;
+   extern server_ExportedSymbols* server_symbols(void);
+```
+
+ So every class instance is represented with a single element structure, encapsulating stable pointer to an instance.
+ Once no longer needed, `DisposeStablePointer()` with that stable pointer shall be called, and if value is not stored
+ somewhere else - it is disposed. For primitive types and `kotlin.String` smart bridges converting to C primitive types
+ or to C strings (which has to be manually freed with `DisposeString()`) are implemented.
+
+ For example, running constructor of class Server taking a string will look like
+
+    server_kref_demo_Server server = server_symbols()->kotlin.demo.Server.Server("the server");
+
+ And disposing no longer needed instance will look like
+
+    server_symbols()->DisposeStablePointer(server.pinned);
+
+ To make code easier readable, macro definitions like
+
+    #define T_(name) server_kref_demo_ ## name
+    #define __ server_symbols()->
+
+ will transform above, overly verbose lines to more readable
+
+    T_(Server) server = __ kotlin.demo.Server.Server("the server");
+
+ `_type()` function will return opaque type pointer, which could be checked with `IsInstance()` operation, like
+
+    __ IsInstance(ref.pinned, __ kotlin.demo.Server._type())

samples/python_extension/build.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+DIR=$(cd "$(dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+source "$DIR/../konan.sh"
+
+PYTHON=python
+
+konanc -p dynamic src/main/kotlin/Server.kt -o server
+sudo $PYTHON src/main/python/setup.py install

samples/python_extension/src/main/c/kotlin_bridge.c

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <Python.h>
+
+#include "server_api.h"
+
+#define __ server_symbols()->
+#define T_(name) server_kref_demo_ ## name
+
+// Note, that as we cache this in the global, and Kotlin/Native object references
+// are currently thread local, we make this global a TLS variable.
+__thread static server_kref_demo_Server server = { 0 };
+
+static T_(Server) getServer() {
+  if (!server.pinned) {
+    server = __ kotlin.demo.Server.Server("the server");
+  }
+  return server;
+}
+
+static server_kref_demo_Session getSession(PyObject* args) {
+   T_(Session) result = { 0 };
+   long long pinned;
+   if (PyArg_ParseTuple(args, "L", &pinned)) {
+       result.pinned = (void*)(uintptr_t)pinned;
+   }
+   return result;
+}
+
+static PyObject* open_session(PyObject* self, PyObject* args) {
+    PyObject *result = NULL;
+    char* string_arg = NULL;
+    int int_arg = 0;
+    if (PyArg_ParseTuple(args, "is", &int_arg, &string_arg)) {
+        server_kref_demo_Session session = __ kotlin.demo.Session.Session(string_arg, int_arg);
+        result = Py_BuildValue("L", session.pinned);
+    }
+    return result;
+}
+
+static PyObject* close_session(PyObject* self, PyObject* args) {
+    T_(Session) session = getSession(args);
+    __ DisposeStablePointer(session.pinned);
+    __ DisposeStablePointer(getServer().pinned);
+    server.pinned = 0;
+    return Py_BuildValue("L", 0);
+}
+
+static PyObject* greet_server(PyObject* self, PyObject* args) {
+    T_(Server) server = getServer();
+    T_(Session) session = getSession(args);
+    const char* string = __ kotlin.demo.Server.greet(server, session);
+    PyObject* result = Py_BuildValue("s", string);
+    __ DisposeString(string);
+    return result;
+}
+
+static PyObject* concat_server(PyObject* self, PyObject* args) {
+    long long session_arg;
+    char* string_arg1 = NULL;
+    char* string_arg2 = NULL;
+    PyObject* result = NULL;
+
+    if (PyArg_ParseTuple(args, "Lss", &session_arg, &string_arg1, &string_arg2)) {
+       T_(Server) server = getServer();
+       T_(Session) session = { (void*)(uintptr_t)session_arg };
+       const char* string = __ kotlin.demo.Server.concat(server, session, string_arg1, string_arg2);
+       result = Py_BuildValue("s", string);
+       __ DisposeString(string);
+    } else {
+        result = Py_BuildValue("s", NULL);
+    }
+    return result;
+}
+
+static PyObject* add_server(PyObject* self, PyObject* args) {
+    long long session_arg;
+    int int_arg1 = 0;
+    int int_arg2 = 0;
+    PyObject* result = NULL;
+
+    if (PyArg_ParseTuple(args, "Lii", &session_arg, &int_arg1, &int_arg2)) {
+       T_(Server) server = getServer();
+       T_(Session) session = { (void*)(uintptr_t)session_arg };
+       int sum = __ kotlin.demo.Server.add(server, session, int_arg1, int_arg2);
+       result = Py_BuildValue("i", sum);
+    } else {
+        result = Py_BuildValue("i", 0);
+    }
+    return result;
+}
+
+static PyMethodDef kotlin_bridge_funcs[] = {
+   { "open_session", (PyCFunction)open_session, METH_VARARGS, "Opens a session" },
+   { "close_session", (PyCFunction)close_session, METH_VARARGS, "Closes the session" },
+   { "greet_server", (PyCFunction)greet_server, METH_VARARGS, "Greeting service" },
+   { "concat_server", (PyCFunction)concat_server, METH_VARARGS, "Concatenation service" },
+   { "add_server", (PyCFunction)add_server, METH_VARARGS, "Addition service" },
+   { NULL }
+};
+
+void initkotlin_bridge(void) {
+   Py_InitModule3("kotlin_bridge", kotlin_bridge_funcs, "Kotlin/Native example module");
+}

samples/python_extension/src/main/kotlin/Server.kt

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package demo
+
+class Session(val name: String, val number: Int)
+
+class Server(val prefix: String) {
+    fun greet(session: Session) = "$prefix: Hello from Kotlin/Native in ${session}"
+    fun concat(session: Session, a: String, b: String) = "$prefix: $a $b in ${session}"
+    fun add(session: Session, a: Int, b: Int) = a + b + session.number
+}

samples/python_extension/src/main/python/main.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python
+#
+# Copyright 2010-2017 JetBrains s.r.o.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import kotlin_bridge
+
+session = kotlin_bridge.open_session(239, 'konan')
+
+message = kotlin_bridge.greet_server(session)
+print "Greet '%s'" % (message)
+
+message = kotlin_bridge.concat_server(session, "Coding", "fun")
+print "Concat '%s'" % (message)
+
+message = kotlin_bridge.add_server(session, 1, 60)
+print "Sum '%d'" % (message)
+
+
+kotlin_bridge.close_session(session)
+

samples/python_extension/src/main/python/setup.py

@@ -0,0 +1,44 @@
+#!/usr/bin/python
+#
+# Copyright 2010-2017 JetBrains s.r.o.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from distutils.core import setup, Extension
+
+setup(name='kotlin_bridge',
+      version='1.0',
+      maintainer = 'JetBrains',
+      maintainer_email = 'info@jetbrains.com',
+      author = 'JetBrains',
+      author_email = 'info@jetbrains.com',
+      description = 'Kotlin/Native Python bridge',
+      long_description = 'Using Kotlin/Native from Python example',
+
+      # data_files=[("/Library/Python/2.7/site-packages/", ['libserver.dylib'])],
+
+      ext_modules=[
+         Extension('kotlin_bridge',
+            include_dirs = ['.'],
+            libraries = ['server'],
+            library_dirs = ['.'],
+            depends = ['server_api.h'],
+            sources = ['src/main/c/kotlin_bridge.c']
+        )
+      ]
+)
+
+# On macOS, after install you may want to copy libserver.dylib to Python's extension directory,
+# and do something like
+#  sudo install_name_tool /Library/Python/2.7/site-packages/kotlin_bridge.so -change libserver.dylib @loader_path/libserver.dylib
+# This way libserver.dylib could be loaded from extension's directory.

samples/settings.gradle

@@ -5,6 +5,7 @@ include ':gtk'
 include ':libcurl'
 include ':nonBlockingEchoServer'
 include ':opengl'
+include ':python_extension'
 include ':socket'
 include ':tetris'
 include ':tensorflow'

shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanTarget.kt

@@ -62,7 +62,7 @@ enum class CompilerOutputKind {
     },
     DYNAMIC {
         override fun suffix(target: KonanTarget?) = ".${target!!.family.dynamicSuffix}"
-        override fun prefix(target: KonanTarget?) = ".${target!!.family.dynamicPrefix}"
+        override fun prefix(target: KonanTarget?) = "${target!!.family.dynamicPrefix}"
     },
     FRAMEWORK {
         override fun suffix(target: KonanTarget?): String = ".framework"