Add Windows i18n support (#617)

backend.native/build.gradle

@@ -157,8 +157,9 @@ targetList.each { target ->
         classpath = project.configurations.cli_bc
         jvmArgs "-ea", 
             "-Dkonan.home=${project.parent.file('dist')}",
-            "-Djava.library.path=${project.buildDir}/nativelibs"
+            "-Djava.library.path=${project.buildDir}/nativelibs",
-        args('-output', project(':runtime').file("build/${target}Stdlib"), 
+            "-Dfile.encoding=UTF-8"
+        args('-output', project(':runtime').file("build/${target}Stdlib"),
             '-nopack', '-nostdlib','-ea',
             '-produce', 'library', '-target', target,
             '-runtime', project(':runtime').file("build/${target}/runtime.bc"),

backend.native/tests/build.gradle

@@ -367,17 +367,14 @@ task hello3(type: RunKonanTest) {
 }
 
 task hello4(type: RunKonanTest) {
-    if (isWindows()) {
-        // To be investigated.
-        disabled = true
-    }
     goldValue = "Hello\nПока\n"
     source = "runtime/basic/hello4.kt"
 }
 
 task tostring0(type: RunKonanTest) {
     if (isWindows()) {
-        // To be investigated.
+        // ト is misprinted for now (as 0x3f).
+        // TODO: investigate.
         disabled = true
     }
     goldValue = "127\n-1\n239\nA\nЁ\nト\n1122334455\n112233445566778899\n3.14159265358\n1.0E27\n1.0E-300\ntrue\nfalse\n"
@@ -395,10 +392,6 @@ task tostring2(type: RunKonanTest) {
 }
 
 task tostring3(type: RunKonanTest) {
-    if (isWindows()) {
-        // Double.toString()
-        disabled = true
-    }
     goldValue = "-128\n127\n-32768\n32767\n" +
             "-2147483648\n2147483647\n-9223372036854775808\n9223372036854775807\n" +
             "1.4E-45\n3.4028235E38\n-Infinity\nInfinity\n" +
@@ -1223,19 +1216,11 @@ task moderately_large_array1(type: RunKonanTest) {
 }
 
 task string_builder0(type: RunKonanTest) {
-    if (isWindows()) {
-        // To be investigated.
-        disabled = true
-    }
     goldValue = "OK\n"
     source = "runtime/text/string_builder0.kt"
 }
 
 task string0(type: RunKonanTest) {
-    if (isWindows()) {
-        // To be investigated.
-        disabled = true
-    }
     goldValue = "true\ntrue\nПРИВЕТ\nпривет\nПока\ntrue\n"
     source = "runtime/text/string0.kt"
 }

build.gradle

@@ -114,7 +114,8 @@ void setupCompilationFlags() {
         ext.host = "mingw"
         ext.targetArgs <<
                 [(ext.host):
-                         ["-target", "x86_64-w64-mingw32", "--sysroot=${mingwWithLlvmDir}", "-DOMIT_BACKTRACE"]]
+                         ["-target", "x86_64-w64-mingw32", "--sysroot=${mingwWithLlvmDir}",
+                          "-DOMIT_BACKTRACE", "-DKONAN_WINDOWS=1"]]
     } else {
          ext.host = "macbook"
          ext.targetArgs <<

buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy

@@ -106,7 +106,7 @@ abstract class KonanTest extends JavaExec {
             errorOutput = log
             super.exec()
         } finally {
-            def logString = log.toString()
+            def logString = log.toString("UTF-8")
             project.file("${output}.compilation.log").write(logString)
             println(logString)
         }
@@ -231,15 +231,16 @@ fun handleExceptionContinuation(x: (Throwable) -> Unit): Continuation<Any?> = ob
 
             ignoreExitValue = true
         }
-        println(out.toString())
+        def result = out.toString("UTF-8")
+        println(result)
 
         if (execResult.exitValue != expectedExitStatus) {
             throw new TestFailedException(
                     "Test failed. Expected exit status: $expectedExitStatus, actual: ${execResult.exitValue}")
         }
 
-        if (goldValue != null && goldValue != out.toString().replace(System.lineSeparator(), "\n")) {
+        if (goldValue != null && goldValue != result.replace(System.lineSeparator(), "\n")) {
-            throw new TestFailedException("Test failed. Expected output: $goldValue, actual output: ${out.toString()}")
+            throw new TestFailedException("Test failed. Expected output: $goldValue, actual output: $result")
         }
     }
 }
@@ -290,7 +291,7 @@ class RunDriverKonanTest extends KonanTest {
             standardOutput = log
             errorOutput = log
         }
-        def logString = log.toString()
+        def logString = log.toString("UTF-8")
         project.file("${output}.compilation.log").write(logString)
         println(logString)
     }

cmd/cinterop

@@ -66,7 +66,8 @@ KONAN_HOME="$(findHome)"
 NATIVE_LIB="${KONAN_HOME}/konan/nativelib"
 JAVA_OPTS="-ea \
     -Djava.library.path=${NATIVE_LIB} \
-    -Dkonan.home=${KONAN_HOME}"
+    -Dkonan.home=${KONAN_HOME} \
+    -Dfile.encoding=UTF-8"
 
 STUB_GENERATOR_JAR="${KONAN_HOME}/konan/lib/StubGenerator.jar"
 KOTLIN_JAR="${KONAN_HOME}/konan/lib/kotlin-compiler.jar"

cmd/konanc

@@ -70,7 +70,7 @@ HELPERS_JAR="${KONAN_HOME}/konan/lib/helpers.jar"
 NATIVE_LIB="${KONAN_HOME}/konan/nativelib"
 KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$KONAN_JAR:$HELPERS_JAR"
 KONAN_COMPILER=org.jetbrains.kotlin.cli.bc.K2NativeKt
-JAVA_OPTS=-ea
+JAVA_OPTS="-ea -Dfile.encoding=UTF-8"
 
 #
 # KONAN BACKEND INVOCATION

runtime/src/main/cpp/Interop.cpp

@@ -46,7 +46,7 @@ ffi_type* convertFfiTypeKindToType(FfiTypeKind typeKind) {
         case FFI_TYPE_KIND_DOUBLE: return &ffi_type_double;
         case FFI_TYPE_KIND_POINTER: return &ffi_type_pointer;
 
-        default: assert(false);
+        default: assert(false); return nullptr;
     }
 }
 

runtime/src/main/cpp/Runtime.cpp

@@ -15,6 +15,9 @@
  */
 
 #include <stdio.h>
+#if KONAN_WINDOWS
+#include <windows.h>
+#endif
 
 #include "Memory.h"
 #include "Runtime.h"
@@ -64,6 +67,12 @@ RuntimeState* InitRuntime() {
   result->memoryState = InitMemory();
   // Keep global variables in state as well.
   InitGlobalVariables();
+#if KONAN_WINDOWS
+  // Note that this code enforces UTF-8 console output, so we may want to rethink
+  // how we perform console IO, if it turns out, that UTF-16 is better output format.
+  SetConsoleCP(CP_UTF8);
+  SetConsoleOutputCP(CP_UTF8);
+#endif
   return result;
 }
 

samples/csvparser/build.sh

@@ -7,6 +7,7 @@ if [ x$TARGET == x ]; then
 case "$OSTYPE" in
   darwin*)  TARGET=macbook ;;
   linux*)   TARGET=linux ;;
+  msys*)    TARGET=mingw ;;
   *)        echo "unknown: $OSTYPE" && exit 1;;
 esac
 fi