added a simple little CompilerPlugin interface and a loosely coupled loader of KDoc if its configured (via -docOutput) and its on the classpath then it can be used at the same stage as a compile & refactored KDoc to work with the compiler. Added a "docStdlib" goal to try out the kdoc on stdlib if the kdoc plugin has been built in kdoc/target/*.jar

build.xml

@@ -48,12 +48,29 @@
     </target>
 
     <target name="compileStdlib" depends="jar">
+        <mkdir dir="${output}/classes/stdlib"/>
+        <java classname="org.jetbrains.jet.cli.KotlinCompiler" failonerror="true">
+            <classpath>
+                <path refid="classpath"/>
+                <pathelement location="${kotlin-home}/lib/kotlin-compiler.jar"/>
+            </classpath>
+            <arg value="-src"/>
+            <arg value="${basedir}/stdlib/ktSrc"/>
+            <arg value="-output"/>
+            <arg value="${output}/classes/stdlib"/>
+        </java>
+    </target>
+
+    <target name="docStdlib">
         <mkdir dir="${output}/classes/stdlib"/>
         <mkdir dir="${output}/apidoc/stdlib"/>
         <java classname="org.jetbrains.jet.cli.KotlinCompiler" failonerror="true">
             <classpath>
                 <path refid="classpath"/>
                 <pathelement location="${kotlin-home}/lib/kotlin-compiler.jar"/>
+                <fileset dir="${basedir}/kdoc/target">
+                  <include name="**/*.jar"/>
+                </fileset>
             </classpath>
             <arg value="-src"/>
             <arg value="${basedir}/stdlib/ktSrc"/>
@@ -70,13 +87,6 @@
             <classpath>
               <path refid="classpath"/>
               <pathelement location="${kotlin-home}/lib/kotlin-compiler.jar"/>
-<!--
-              <pathelement location="${kotlin-home}/lib/kotlin-runtime.jar"/>
-              <pathelement location="${basedir}/testlib/lib/junit-4.9.jar"/>
-              <fileset dir="${basedir}/testlib/lib">
-                <include name="**/*.jar"/>
-              </fileset>
--->
             </classpath>
             <arg value="-stdlib"/>
             <arg value="${kotlin-home}/lib/kotlin-runtime.jar"/>

compiler/cli/src/org/jetbrains/jet/cli/KDocProcessor.java

@@ -17,38 +17,30 @@
  */
 package org.jetbrains.jet.cli;
 
-import com.google.common.base.Predicates;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
 import com.intellij.lang.ASTNode;
 import com.intellij.openapi.Disposable;
-import com.intellij.openapi.project.Project;
 import com.intellij.psi.PsiElement;
-import com.intellij.psi.PsiFile;
 import org.jetbrains.annotations.Nullable;
+import org.jetbrains.jet.compiler.CompilerPlugin;
 import org.jetbrains.jet.compiler.JetCoreEnvironment;
-import org.jetbrains.jet.compiler.JetFileProcessor;
-import org.jetbrains.jet.lang.Configuration;
-import org.jetbrains.jet.lang.cfg.pseudocode.JetControlFlowDataTraceFactory;
 import org.jetbrains.jet.lang.descriptors.ClassDescriptor;
 import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor;
-import org.jetbrains.jet.lang.descriptors.NamespaceDescriptor;
 import org.jetbrains.jet.lang.psi.JetFile;
 import org.jetbrains.jet.lang.psi.JetPsiFactory;
-import org.jetbrains.jet.lang.resolve.AnalyzingUtils;
 import org.jetbrains.jet.lang.resolve.BindingContext;
-import org.jetbrains.jet.lang.resolve.BindingTraceContext;
-import org.jetbrains.jet.lang.resolve.java.JavaBridgeConfiguration;
 import org.jetbrains.jet.lexer.JetTokens;
 import org.jetbrains.jet.resolve.DescriptorRenderer;
 
+import java.io.File;
+import java.lang.reflect.Constructor;
 import java.util.Collection;
 import java.util.List;
-import java.util.Set;
 
 /**
+ * A simple facade to auto-detect the KDoc processor if its available on the classpath
  */
-public class KDocProcessor implements JetFileProcessor {
+public class KDocProcessor implements CompilerPlugin {
 
     private final String outputDir;
 
@@ -56,11 +48,50 @@ public class KDocProcessor implements JetFileProcessor {
         this.outputDir = outputDir;
     }
 
+    public CompilerPlugin createCompilerPlugin() {
+        // lets see if we can see the KDoc class
+        String name = "org.jetbrains.kotlin.doc.KDoc";
+        Class<?> aClass = null;
+        try {
+            aClass = loadClass(name);
+        } catch (ClassNotFoundException e) {
+            System.out.println("Could not find class: " + name);
+            return null;
+        }
+        if (aClass != null) {
+            try {
+                File dir = new File(outputDir);
+                Constructor<?> constructor = aClass.getConstructor(File.class);
+                if (constructor != null) {
+                    return (CompilerPlugin) constructor.newInstance(dir);
+                }
+            } catch (Exception e) {
+                System.out.println("Failed to create Processor: " + e);
+            }
+        }
+        return null;
+    }
+
+    public static Class<?> loadClass(String name) throws ClassNotFoundException {
+        try {
+            return Class.forName(name);
+        } catch (ClassNotFoundException e) {
+            try {
+                return Thread.currentThread().getContextClassLoader().loadClass(name);
+            } catch (ClassNotFoundException e1) {
+                return KDocProcessor.class.getClassLoader().loadClass(name);
+            }
+        }
+    }
+
+
     @Override
-    public void processFiles(List<JetFile> sources) {
+    public void processFiles(BindingContext context, List<JetFile> sources) {
+        /*
         // JetFile's are PSI (Program Source Interface) classes, i.e. they contain the concrete syntax trees of files
         if (sources.isEmpty()) return;
 
+
         // Let's perform the semantic analysis
         Project project = sources.get(0).getProject();
         Configuration javaBridgeConfiguration = JavaBridgeConfiguration.createJavaBridgeConfiguration(project, new BindingTraceContext(), Configuration.EMPTY);
@@ -82,6 +113,7 @@ public class KDocProcessor implements JetFileProcessor {
         }
 
         // TODO fire up the KDoc processor here...
+        */
     }
 
     private void processDescriptors(Collection<DeclarationDescriptor> allDescriptors, BindingContext context) {
@@ -125,6 +157,6 @@ public class KDocProcessor implements JetFileProcessor {
         });
         List<JetFile> files = Lists.newArrayList();
         files.add(JetPsiFactory.createFile(jetCoreEnvironment.getProject(), "package a;/**sdfsdf*/class A {fun foo() {} /**doc*/ fun bar() {}}"));
-        new KDocProcessor("").processFiles(files);
+        new KDocProcessor("").processFiles(null, files);
     }
 }

compiler/cli/src/org/jetbrains/jet/cli/KotlinCompiler.java

@@ -21,6 +21,7 @@ import com.sampullara.cli.Argument;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.jet.compiler.CompileEnvironment;
 import org.jetbrains.jet.compiler.CompileEnvironmentException;
+import org.jetbrains.jet.compiler.CompilerPlugin;
 import org.jetbrains.jet.compiler.FileNameTransformer;
 
 import java.io.ByteArrayOutputStream;
@@ -127,9 +128,13 @@ public class KotlinCompiler {
                 environment.setErrorStream(errStream);
             }
 
-//            if (arguments.docOutputDir != null) {
+            if (arguments.docOutputDir != null) {
-//                environment.getMyEnvironment().getFileProcessors().add(new KDocProcessor(arguments.docOutputDir));
+                KDocProcessor factory = new KDocProcessor(arguments.docOutputDir);
-//            }
+                CompilerPlugin processor = factory.createCompilerPlugin();
+                if (processor != null) {
+                    environment.getMyEnvironment().getCompilerPlugins().add(processor);
+                }
+            }
 
             if (arguments.stdlib != null) {
                 environment.setStdlib(arguments.stdlib);

compiler/cli/src/org/jetbrains/jet/compiler/CompileSession.java

@@ -204,10 +204,10 @@ public class CompileSession {
         generationState.compileCorrectFiles(myBindingContext, mySourceFiles);
         ClassFileFactory answer = generationState.getFactory();
 
-        List<JetFileProcessor> fileProcessors = myEnvironment.getFileProcessors();
+        List<CompilerPlugin> fileProcessors = myEnvironment.getCompilerPlugins();
         if (fileProcessors != null) {
-            for (JetFileProcessor processor : fileProcessors) {
+            for (CompilerPlugin processor : fileProcessors) {
-                processor.processFiles(getSourceFileNamespaces());
+                processor.processFiles(myBindingContext, getSourceFileNamespaces());
             }
         }
         return answer;

compiler/cli/src/org/jetbrains/jet/compiler/CompilerPlugin.java

@@ -18,11 +18,14 @@
 package org.jetbrains.jet.compiler;
 
 import org.jetbrains.jet.lang.psi.JetFile;
+import org.jetbrains.jet.lang.resolve.BindingContext;
 
 import java.util.List;
 
 /**
+ * A simple interface for compiler plugins to run after the compiler has finished such as for things like
+ * generating documentation or code generation etc
  */
-public interface JetFileProcessor {
+public interface CompilerPlugin {
-    void processFiles(List<JetFile> sources);
+    void processFiles(BindingContext context, List<JetFile> sources);
 }

compiler/cli/src/org/jetbrains/jet/compiler/JetCoreEnvironment.java

@@ -33,7 +33,7 @@ import java.util.List;
  * @author yole
  */
 public class JetCoreEnvironment extends JavaCoreEnvironment {
-    private List<JetFileProcessor> fileProcessors = new ArrayList<JetFileProcessor>();
+    private List<CompilerPlugin> compilerPlugins = new ArrayList<CompilerPlugin>();
 
     public JetCoreEnvironment(Disposable parentDisposable) {
         super(parentDisposable);
@@ -61,11 +61,11 @@ public class JetCoreEnvironment extends JavaCoreEnvironment {
         return myApplication;
     }
 
-    public List<JetFileProcessor> getFileProcessors() {
+    public List<CompilerPlugin> getCompilerPlugins() {
-        return fileProcessors;
+        return compilerPlugins;
     }
 
-    public void setFileProcessors(List<JetFileProcessor> fileProcessors) {
+    public void setCompilerPlugins(List<CompilerPlugin> compilerPlugins) {
-        this.fileProcessors = fileProcessors;
+        this.compilerPlugins = compilerPlugins;
     }
 }

kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/KDoc.kt

@@ -5,8 +5,67 @@ import org.jetbrains.kotlin.template.TextTemplate
 import org.jetbrains.kotlin.model.KModel
 
 import java.io.File
+import org.jetbrains.jet.lang.resolve.BindingContext
+import org.jetbrains.jet.compiler.CompilerPlugin
+import java.util.List
+import org.jetbrains.jet.lang.psi.JetFile
+import org.jetbrains.jet.lang.descriptors.NamespaceDescriptor
+import java.util.HashSet
+import com.intellij.psi.PsiElement
+import org.jetbrains.jet.lexer.JetTokens
 
-class KDocProcessor(val model: KModel, val outputDir: File) {
+class KDoc(val outputDir: File) : KDocSupport() {
+    val model = KModel()
+
+    /** TODO compile errors so lets use Java for this bit for now...
+    override fun processFiles(context: BindingContext, files: List<JetFile?>?) {
+        println("==== KDoc running! Generating to $outputDir")
+        if (context != null) {
+            val namespaces = HashSet<NamespaceDescriptor>()
+            if (files != null) {
+                for (source in files) {
+                    if (source != null) {
+                        val namespaceDescriptor = context.get(BindingContext.NAMESPACE, source)
+                        if (namespaceDescriptor != null) {
+                            namespaces.add(namespaceDescriptor)
+                        }
+                    }
+                }
+            }
+        }
+    }
+    */
+
+    override fun generate() {
+        for (classElement in allClasses) {
+            if (classElement != null) {
+                //val docComment = getDocCommentFor(classElement.sure()) ?: "";
+                val name = classElement?.getName()
+                if (name != null) {
+                    println("Found class: ${name}")
+                    //model.getClass(name)
+                }
+            }
+        }
+        if (!model.packages.isEmpty()) {
+            val generator = KDocGenerator(model, outputDir)
+            generator.execute()
+        }
+    }
+
+    protected fun getDocCommentFor(psiElement: PsiElement): String? {
+        // This method is a hack. Doc comments should be easily accessible, but they aren't for now.
+        var node = psiElement.getNode()?.getTreePrev()
+        while (node != null && (node?.getElementType() == JetTokens.WHITE_SPACE || node?.getElementType() == JetTokens.BLOCK_COMMENT)) {
+            node = node?.getTreePrev();
+        }
+        if (node == null) return null;
+        if (node?.getElementType() != JetTokens.DOC_COMMENT) return null;
+        return node?.getText();
+    }
+}
+
+class KDocGenerator(val model: KModel, val outputDir: File) {
 
     fun execute(): Unit {
         run("allclasses-frame.html", AllClassesFrameTemplate(model, " target=\"classFrame\""))

kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/KDocSupport.java

@@ -0,0 +1,84 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jetbrains.kotlin.doc;
+
+import com.intellij.psi.PsiElement;
+import org.jetbrains.jet.compiler.CompilerPlugin;
+import org.jetbrains.jet.lang.descriptors.ClassDescriptor;
+import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor;
+import org.jetbrains.jet.lang.descriptors.NamespaceDescriptor;
+import org.jetbrains.jet.lang.psi.JetFile;
+import org.jetbrains.jet.lang.resolve.BindingContext;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * TODO This class is written in Java for now to work around a few gremlins in Kotlin...
+ */
+public abstract class KDocSupport implements CompilerPlugin {
+    protected Set<NamespaceDescriptor> allNamespaces = new HashSet<NamespaceDescriptor>();
+    protected Set<ClassDescriptor> allClasses = new HashSet<ClassDescriptor>();
+
+    public void processFiles(BindingContext context, List<JetFile> sources) {
+        for (JetFile source : sources) {
+            // We retrieve a descriptor by a PSI element from the context
+            NamespaceDescriptor namespaceDescriptor = context.get(BindingContext.NAMESPACE, source);
+            if (namespaceDescriptor != null) {
+                allNamespaces.add(namespaceDescriptor);
+            }
+        }
+
+        for (NamespaceDescriptor namespace : allNamespaces) {
+            // Let's take all the declarations in the namespace...
+            processDescriptors(namespace.getMemberScope().getAllDescriptors(), context);
+        }
+
+        generate();
+    }
+
+    protected abstract void generate();
+
+    private void processDescriptors(Collection<DeclarationDescriptor> allDescriptors, BindingContext context) {
+        for (DeclarationDescriptor descriptor : allDescriptors) {
+            PsiElement classElement = context.get(BindingContext.DESCRIPTOR_TO_DECLARATION, descriptor);
+/*
+            // Print the doc comment text
+            String docComment = getDocCommentFor(classElement);
+            if (docComment != null) {
+                System.out.println("Docs for " + descriptor.getName() + ": " + docComment);
+            }
+            else {
+                System.out.println("No docs for " + descriptor.getName());
+            }
+            // Print the class header (verbose)
+            System.out.println(DescriptorRenderer.TEXT.render(descriptor));
+*/
+            // Process members, if any
+            if (descriptor instanceof ClassDescriptor) {
+                ClassDescriptor classDescriptor = (ClassDescriptor) descriptor;
+                if (classElement != null) {
+                    allClasses.add(classDescriptor);
+                }
+                //processDescriptors(classDescriptor.getDefaultType().getMemberScope().getAllDescriptors(), context);
+            }
+        }
+    }
+}

kdoc/src/test/kotlin/test/kotlin/doc/ModelTest.kt

@@ -9,7 +9,7 @@ import junit.framework.TestCase
 
 // TODO should use the ktest library really for nicer asserts
 import junit.framework.Assert.assertEquals
-import org.jetbrains.kotlin.doc.KDocProcessor
+import org.jetbrains.kotlin.doc.KDocGenerator
 import java.io.File
 
 class ModelTest : TestCase() {
@@ -56,7 +56,7 @@ class ModelTest : TestCase() {
             println()
         }
 
-        val processor = KDocProcessor(model, File("target/test-data/ModelTest"))
+        val processor = KDocGenerator(model, File("target/test-data/ModelTest"))
         processor.execute()
     }
 }