KotlinCompiler and tests aware on standard library

build.xml

@@ -9,6 +9,11 @@
         <pathelement path="${output}/classes/runtime"/>
     </path>
 
+    <path id="classpath.kotlin">
+        <path refid="classpath"/>
+        <pathelement path="${output}/classes/compiler"/>
+    </path>
+
     <path id="sourcepath">
         <dirset dir="${basedir}/compiler">
             <include name="frontend/src"/>
@@ -26,9 +31,21 @@
         </javac>
     </target>
 
-    <target name="jarRT" depends="compileRT">
+    <target name="compileStdlib" depends="compile">
+        <mkdir dir="${output}/classes/stdlib"/>
+        <java classname="org.jetbrains.jet.cli.KotlinCompiler">
+            <classpath refid="classpath.kotlin"/>
+            <arg value="-src"/>
+            <arg value="${basedir}/stdlib/ktSrc"/>
+            <arg value="-output"/>
+            <arg value="${basedir}/classes/stdlib"/>
+        </java>
+    </target>
+
+    <target name="jarRT" depends="compileStdlib">
         <jar destfile="${output}/kotlin-runtime.jar">
             <fileset dir="${output}/classes/runtime"/>
+            <fileset dir="${output}/classes/stdlib"/>
             <fileset dir="${basedir}/stdlib/ktSrc"/>
         </jar>
     </target>
@@ -49,7 +66,11 @@
         </jar>
     </target>
 
-    <target name="dist" depends="jarRT,jar">
+    <target name="clean">
+        <delete dir="${output}"/>
+    </target>
+
+    <target name="dist" depends="clean,jarRT,jar">
         <zip destfile="${output}/${output.name}.zip">
             <zipfileset prefix="${output.name}/bin" filemode="755" dir="${basedir}/compiler/cli/bin"/>
             <zipfileset prefix="${output.name}/lib" dir="${basedir}/ideaSDK"/>

compiler/backend/src/org/jetbrains/jet/compiler/CompileEnvironment.java

@@ -5,7 +5,9 @@ import com.intellij.openapi.application.PathManager;
 import com.intellij.openapi.util.Disposer;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.psi.PsiFile;
 import com.intellij.util.Function;
 import com.intellij.util.Processor;
 import jet.modules.IModuleBuilder;
@@ -22,6 +24,8 @@ import java.io.*;
 import java.lang.reflect.Method;
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.jar.*;
 
@@ -53,14 +57,18 @@ public class CompileEnvironment {
     }
 
     public boolean initializeKotlinRuntime() {
+        return initializeKotlinRuntime(myEnvironment);
+    }
+
+    public static boolean initializeKotlinRuntime(JetCoreEnvironment environment) {
         final File unpackedRuntimePath = getUnpackedRuntimePath();
         if (unpackedRuntimePath != null) {
-            myEnvironment.addToClasspath(unpackedRuntimePath);
+            environment.addToClasspath(unpackedRuntimePath);
         }
         else {
             final File runtimeJarPath = getRuntimeJarPath();
             if (runtimeJarPath != null && runtimeJarPath.exists()) {
-                myEnvironment.addToClasspath(runtimeJarPath);
+                environment.addToClasspath(runtimeJarPath);
             }
             else {
                 return false;
@@ -147,7 +155,7 @@ public class CompileEnvironment {
         return null;
     }
 
-    public void compileModuleScript(String moduleFile) {
+    public void compileModuleScript(String moduleFile, String jarPath, boolean jarRuntime) {
         final IModuleSetBuilder moduleSetBuilder = loadModuleScript(moduleFile);
         if (moduleSetBuilder == null) {
             return;
@@ -156,9 +164,9 @@ public class CompileEnvironment {
         final String directory = new File(moduleFile).getParent();
         for (IModuleBuilder moduleBuilder : moduleSetBuilder.getModules()) {
             ClassFileFactory moduleFactory = compileModule(moduleBuilder, directory);
-            final String path = new File(directory, moduleBuilder.getModuleName() + ".jar").getPath();
+            final String path = jarPath != null ? jarPath : new File(directory, moduleBuilder.getModuleName() + ".jar").getPath();
             try {
-                writeToJar(moduleFactory, new FileOutputStream(path), null, true);
+                writeToJar(moduleFactory, new FileOutputStream(path), null, jarRuntime);
             } catch (FileNotFoundException e) {
                 throw new CompileEnvironmentException("Invalid jar path " + path, e);
             }
@@ -168,25 +176,7 @@ public class CompileEnvironment {
     public IModuleSetBuilder loadModuleScript(String moduleFile) {
         CompileSession scriptCompileSession = new CompileSession(myEnvironment);
         scriptCompileSession.addSources(moduleFile);
-
+        scriptCompileSession.addStdLibSources();
-        URL url = CompileEnvironment.class.getClassLoader().getResource("ModuleBuilder.kt");
-        if (url != null) {
-            String path = url.getPath();
-            if (path.startsWith("file:")) {
-                path = path.substring(5);
-            }
-            final VirtualFile vFile = myEnvironment.getJarFileSystem().findFileByPath(path);
-            if (vFile == null) {
-                throw new CompileEnvironmentException("Couldn't load ModuleBuilder.kt from runtime jar: "+ url);
-            }
-            scriptCompileSession.addSources(vFile);
-        }
-        else {
-            // building from source
-            final String homeDirectory = getHomeDirectory();
-            final File file = new File(homeDirectory, "stdlib/ktSrc/ModuleBuilder.kt");
-            scriptCompileSession.addSources(myEnvironment.getLocalFileSystem().findFileByPath(file.getPath()));
-        }
 
         if (!scriptCompileSession.analyze(myErrorStream)) {
             return null;
@@ -223,6 +213,7 @@ public class CompileEnvironment {
 
     public ClassFileFactory compileModule(IModuleBuilder moduleBuilder, String directory) {
         CompileSession moduleCompileSession = new CompileSession(myEnvironment);
+        moduleCompileSession.addStdLibSources();
         for (String sourceFile : moduleBuilder.getSourceFiles()) {
             moduleCompileSession.addSources(new File(directory, sourceFile).getPath());
         }
@@ -249,10 +240,21 @@ public class CompileEnvironment {
                 mainAttributes.putValue("Main-Class", mainClass);
             }
             JarOutputStream stream = new JarOutputStream(fos, manifest);
+            List<String> sanitized = Arrays.asList("kotlin/", "std/");
             try {
                 for (String file : factory.files()) {
-                    stream.putNextEntry(new JarEntry(file));
+                    boolean skip = false;
-                    stream.write(factory.asBytes(file));
+                    for (String prefix : sanitized) {
+                        if(file.startsWith(prefix)) {
+                            skip = true;
+                            break;
+                        }
+                    }
+
+                    if(!skip) {
+                        stream.putNextEntry(new JarEntry(file));
+                        stream.write(factory.asBytes(file));
+                    }
                 }
                 if (includeRuntime) {
                     writeRuntimeToJar(stream);
@@ -316,7 +318,7 @@ public class CompileEnvironment {
         }
     }
 
-    public void compileBunchOfSources(String sourceFileOrDir, String jar, String outputDir) {
+    public void compileBunchOfSources(String sourceFileOrDir, String jar, String outputDir, boolean includeRuntime) {
         CompileSession session = new CompileSession(myEnvironment);
         session.addSources(sourceFileOrDir);
 
@@ -334,7 +336,7 @@ public class CompileEnvironment {
         ClassFileFactory factory = session.generate();
         if (jar != null) {
             try {
-                writeToJar(factory, new FileOutputStream(jar), mainClass, true);
+                writeToJar(factory, new FileOutputStream(jar), mainClass, includeRuntime);
             } catch (FileNotFoundException e) {
                 throw new CompileEnvironmentException("Invalid jar path " + jar, e);
             }
@@ -358,5 +360,4 @@ public class CompileEnvironment {
             }
         }
     }
-
 }

compiler/backend/src/org/jetbrains/jet/compiler/CompileSession.java

@@ -16,6 +16,7 @@ import org.jetbrains.jet.lang.resolve.BindingContext;
 import org.jetbrains.jet.lang.resolve.java.JavaDefaultImports;
 import org.jetbrains.jet.plugin.JetFileType;
 
+import java.io.File;
 import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.List;
@@ -47,7 +48,22 @@ public class CompileSession {
             return;
         }
 
-        addSources(vFile);
+        addSources(new File(path));
+    }
+
+    private void addSources(File file) {
+        if(file.isDirectory()) {
+            for (File child : file.listFiles()) {
+                addSources(child);
+            }
+        }
+        else {
+            VirtualFile fileByPath = myEnvironment.getLocalFileSystem().findFileByPath(file.getAbsolutePath());
+            PsiFile psiFile = PsiManager.getInstance(myEnvironment.getProject()).findFile(fileByPath);
+            if(psiFile instanceof JetFile) {
+                mySourceFileNamespaces.add(((JetFile) psiFile).getRootNamespace());
+            }
+        }
     }
 
     public void addSources(VirtualFile vFile) {
@@ -66,13 +82,6 @@ public class CompileSession {
         }
     }
 
-    public void addLibrarySources(VirtualFile vFile) {
-        PsiFile psiFile = PsiManager.getInstance(myEnvironment.getProject()).findFile(vFile);
-        if (psiFile instanceof JetFile) {
-            myLibrarySourceFileNamespaces.add(((JetFile) psiFile).getRootNamespace());
-        }
-    }
-
     public List<JetNamespace> getSourceFileNamespaces() {
         return mySourceFileNamespaces;
     }
@@ -104,4 +113,22 @@ public class CompileSession {
         generationState.compileCorrectNamespaces(myBindingContext, mySourceFileNamespaces);
         return generationState.createText();
     }
+
+    public boolean addStdLibSources() {
+        final File unpackedRuntimePath = CompileEnvironment.getUnpackedRuntimePath();
+        if (unpackedRuntimePath != null) {
+            addSources(new File(unpackedRuntimePath, "../../../stdlib/ktSrc").getAbsoluteFile());
+        }
+        else {
+            final File runtimeJarPath = CompileEnvironment.getRuntimeJarPath();
+            if (runtimeJarPath != null && runtimeJarPath.exists()) {
+                // todo
+                throw new UnsupportedOperationException("Loading of stdlib sources from jar");
+            }
+            else {
+                return false;
+            }
+        }
+        return true;
+    }
 }

compiler/cli/src/org/jetbrains/jet/cli/KotlinCompiler.java

@@ -23,6 +23,8 @@ public class KotlinCompiler {
         public String src;
         @Argument(value = "module", description = "module to compile")
         public String module;
+        @Argument(value = "includeRuntime", description = "include Kotlin runtime in to resulting jar")
+        public boolean includeRuntime;
     }
 
     public static void main(String[] args) {
@@ -32,7 +34,7 @@ public class KotlinCompiler {
             Args.parse(arguments, args);
         }
         catch (Throwable t) {
-            System.out.println("Usage: KotlinCompiler [-output <outputDir>|-jar <jarFileName>] -src <filename or dirname>");
+            System.out.println("Usage: KotlinCompiler [-output <outputDir>|-jar <jarFileName>] [-src <filename or dirname>|-module <module file>] [-includeRuntime]");
             t.printStackTrace();
             return;
         }
@@ -47,11 +49,11 @@ public class KotlinCompiler {
             }
 
             if (arguments.module != null) {
-                environment.compileModuleScript(arguments.module);
+                environment.compileModuleScript(arguments.module, arguments.jar, arguments.includeRuntime);
                 return;
             }
             else {
-                environment.compileBunchOfSources(arguments.src, arguments.jar, arguments.outputDir);
+                environment.compileBunchOfSources(arguments.src, arguments.jar, arguments.outputDir, arguments.includeRuntime);
             }
         } catch (CompileEnvironmentException e) {
             System.out.println(e.getMessage());

compiler/testData/compiler/smoke/Smoke.kt

@@ -1,4 +1,7 @@
 namespace Smoke
 
+import std.io.*
+
 fun main(args: Array<String>) {
+    print(args)
 }

compiler/tests/compiler-tests.iml

@@ -11,6 +11,7 @@
     <orderEntry type="module" module-name="frontend" />
     <orderEntry type="module" module-name="frontend.java" />
     <orderEntry type="module" module-name="stdlib" />
+    <orderEntry type="module" module-name="cli" />
   </component>
 </module>
 

compiler/tests/org/jetbrains/jet/codegen/CodegenTestCase.java

@@ -1,6 +1,10 @@
 package org.jetbrains.jet.codegen;
 
+import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.openapi.vfs.CharsetToolkit;
+import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.psi.PsiFile;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.jet.JetLiteFixture;
 import org.jetbrains.jet.lang.psi.JetFile;
@@ -9,6 +13,8 @@ import org.jetbrains.jet.lang.resolve.AnalyzingUtils;
 import org.jetbrains.jet.parsing.JetParsingTest;
 import org.junit.Assert;
 
+import java.io.File;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;

compiler/tests/org/jetbrains/jet/codegen/StdlibTest.java

@@ -6,6 +6,7 @@ import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.openapi.vfs.LocalFileSystem;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.psi.PsiFile;
+import org.jetbrains.jet.compiler.CompileEnvironment;
 import org.jetbrains.jet.compiler.CompileSession;
 import org.jetbrains.jet.lang.psi.JetNamespace;
 import org.jetbrains.jet.lang.resolve.AnalyzingUtils;
@@ -30,8 +31,8 @@ public class StdlibTest extends CodegenTestCase {
 
         session.addSources(myFile.getVirtualFile());
         try {
-            session.addSources(addStdLib());
+            session.addStdLibSources();
-        } catch (IOException e) {
+        } catch (Throwable e) {
             throw new RuntimeException(e);
         }
 
@@ -45,9 +46,9 @@ public class StdlibTest extends CodegenTestCase {
     protected ClassFileFactory generateClassesInFile() {
         try {
             CompileSession session = new CompileSession(myEnvironment);
-
+            CompileEnvironment.initializeKotlinRuntime(myEnvironment);
             session.addSources(myFile.getVirtualFile());
-            session.addSources(addStdLib());
+            session.addStdLibSources();
 
             if (!session.analyze(System.out)) {
                 return null;
@@ -57,18 +58,11 @@ public class StdlibTest extends CodegenTestCase {
         } catch (RuntimeException e) {
             System.out.println(generateToText());
             throw e;
-        } catch (IOException e) {
+        } catch (Throwable e) {
             throw new RuntimeException(e);
         }
     }
 
-    private VirtualFile addStdLib() throws IOException {
-        String text = FileUtil.loadFile(new File(JetParsingTest.getTestDataDir() + "/../../stdlib/ktSrc/StandardLibrary.kt"), CharsetToolkit.UTF8).trim();
-        text = StringUtil.convertLineSeparators(text);
-        PsiFile stdLibFile = createFile("StandardLibrary.kt", text);
-        return stdLibFile.getVirtualFile();
-    }
-
     public void testInputStreamIterator () {
         blackBoxFile("inputStreamIterator.jet");
 //        System.out.println(generateToText());
@@ -85,4 +79,16 @@ public class StdlibTest extends CodegenTestCase {
     public void testKt528 () {
         blackBoxFile("regressions/kt528.kt");
     }
+
+    public void testCollectionSize () throws Exception {
+        loadText("import std.util.*; fun box() = if(java.util.Arrays.asList(0, 1, 2)?.size == 3) \"OK\" else \"fail\"");
+//        System.out.println(generateToText());
+        blackBox();
+    }
+
+    public void testCollectionEmpty () throws Exception {
+        loadText("import std.util.*; fun box() = if(java.util.Arrays.asList(0, 1, 2)?.empty ?: false) \"OK\" else \"fail\"");
+//        System.out.println(generateToText());
+        blackBox();
+    }
 }

compiler/tests/org/jetbrains/jet/compiler/CompileEnvironmentTest.java

@@ -3,20 +3,20 @@ package org.jetbrains.jet.compiler;
 import jet.modules.IModuleBuilder;
 import jet.modules.IModuleSetBuilder;
 import junit.framework.TestCase;
+import org.jetbrains.jet.cli.KotlinCompiler;
 import org.jetbrains.jet.codegen.ClassFileFactory;
 import org.jetbrains.jet.parsing.JetParsingTest;
 
-import java.io.ByteArrayInputStream;
+import java.io.*;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.jar.JarEntry;
 import java.util.jar.JarInputStream;
 
 /**
  * @author yole
+ * @author alex.tkachman
  */
 public class CompileEnvironmentTest extends TestCase {
     private CompileEnvironment environment;
@@ -51,6 +51,31 @@ public class CompileEnvironmentTest extends TestCase {
         assertTrue(entries.contains("Smoke/namespace.class"));
     }
 
+    public void testSmokeWithCompiler() throws IOException {
+        File tempFile = File.createTempFile("compilerTest", "compilerTest");
+        try {
+            KotlinCompiler.main(Arrays.asList("-module", JetParsingTest.getTestDataDir() + "/compiler/smoke/Smoke.kts", "-jar", tempFile.getAbsolutePath()).toArray(new String[0]));
+            FileInputStream fileInputStream = new FileInputStream(tempFile);
+            try {
+                JarInputStream is = new JarInputStream(fileInputStream);
+                try {
+                    final List<String> entries = listEntries(is);
+                    assertTrue(entries.contains("Smoke/namespace.class"));
+                    assertEquals(1, entries.size());
+                }
+                finally {
+                    is.close();
+                }
+            }
+            finally {
+                fileInputStream.close();
+            }
+        }
+        finally {
+            tempFile.delete();
+        }
+    }
+    
     private List<String> listEntries(JarInputStream is) throws IOException {
         List<String> entries = new ArrayList<String>();
         while (true) {

stdlib/ktSrc/JavaUtil.kt

@@ -0,0 +1,11 @@
+namespace std
+
+namespace util {
+    import java.util.*
+
+    val Collection<*>.size : Int
+        get() = size()
+
+    val Collection<*>.empty : Boolean
+        get() = isEmpty()
+}

stdlib/ktSrc/StandardLibrary.kt

@@ -1,15 +1,5 @@
 namespace std
 
-namespace util {
-    import java.util.*
-
-    val <T> Collection<T>.size : Int
-        get() = size()
-
-    val <T> Collection<T>.empty : Boolean
-        get() = isEmpty()
-}
-
 namespace io {
     import java.io.*
     import java.nio.charset.*