working on syntax highlighter2 for kdoc

libraries/tools/kdoc/pom.xml

@@ -38,6 +38,15 @@
         <sourceDirectory>src/main/kotlin</sourceDirectory>
         <testSourceDirectory>src/test/kotlin</testSourceDirectory>
 
+        <resources>
+            <resource>
+                <directory>src/main/kotlin</directory>
+                <includes>
+                    <include>**/*.css</include>
+                </includes>
+            </resource>
+        </resources>
+
         <plugins>
             <plugin>
                 <groupId>org.jetbrains.kotlin</groupId>

libraries/tools/kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/KDocCompiler.kt

@@ -8,6 +8,7 @@ import org.jetbrains.jet.cli.common.CLICompiler
 import org.jetbrains.jet.cli.jvm.compiler.K2JVMCompileEnvironmentConfiguration
 import org.jetbrains.kotlin.doc.highlighter.HtmlCompilerPlugin
 import org.jetbrains.jet.cli.common.ExitCode
+import org.jetbrains.kotlin.doc.highlighter2.Html2CompilerPlugin
 
 /**
 * Main for running the KDocCompiler
@@ -32,10 +33,8 @@ class KDocCompiler() : K2JVMCompiler() {
                 kdoc.config = arguments.apply()
             }
             val plugins = configuration.getCompilerPlugins().orEmpty()
-/*
+            val sourcePlugin = Html2CompilerPlugin(arguments as KDocArguments)
-            val sourcePlugin = HtmlCompilerPlugin()
             plugins.add(sourcePlugin)
-*/
             plugins.add(kdoc);
         }
     }

libraries/tools/kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/highlighter2/Html2CompilerPlugin.kt

@@ -0,0 +1,139 @@
+package org.jetbrains.kotlin.doc.highlighter2
+
+import org.jetbrains.jet.cli.common.CompilerPlugin
+import org.jetbrains.jet.cli.common.CompilerPluginContext
+import java.io.File
+import java.util.List
+import org.jetbrains.jet.internal.com.intellij.psi.PsiFile
+import org.jetbrains.jet.internal.com.intellij.openapi.vfs.local.CoreLocalVirtualFile
+import org.jetbrains.jet.cli.jvm.K2JVMCompilerArguments
+import org.jetbrains.kotlin.doc.KDocArguments
+import org.jetbrains.kotlin.template.TextTemplate
+import org.jetbrains.kotlin.template.HtmlTemplate
+import org.jetbrains.kotlin.template.escapeHtml
+import org.jetbrains.jet.internal.com.intellij.psi.PsiElement
+import org.jetbrains.jet.internal.com.intellij.psi.impl.source.tree.LeafPsiElement
+import org.jetbrains.jet.JetNodeTypes
+import org.jetbrains.jet.lexer.JetTokens
+
+
+class Html2CompilerPlugin(private val compilerArguments: KDocArguments) : CompilerPlugin {
+
+    private val docOutputRoot: File
+    {
+        val docOutputDir = compilerArguments.docConfig.docOutputDir
+        if (docOutputDir.isEmpty()) {
+            throw IllegalArgumentException("empty doc output dir")
+        }
+        docOutputRoot = File(docOutputDir)
+    }
+
+    private val srcOutputRoot = File(docOutputRoot, "src")
+
+    private val sourceDirs: List<File> =
+        compilerArguments
+                .getSourceDirs()
+                .orEmpty()
+                .requireNoNulls()
+                .map { path -> File(path).getCanonicalFile()!! }
+
+    private val sourceDirPaths: List<String> = sourceDirs.map { d -> d.getPath()!! }
+
+    private fun fileToWrite(psiFile: PsiFile): String {
+        val file = File((psiFile.getVirtualFile() as CoreLocalVirtualFile).getPath()).getCanonicalFile()!!
+        val filePath = file.getPath()!!
+        for (sourceDirPath in sourceDirPaths) {
+            if (filePath.startsWith(sourceDirPath) && filePath.length() > sourceDirPath.length()) {
+                val relativePath = filePath.substring(sourceDirPath.length + 1)
+                return relativePath
+            }
+        }
+        throw Exception("$file is not a child of any source roots $sourceDirPaths")
+    }
+
+    override fun processFiles(context: CompilerPluginContext) {
+        srcOutputRoot.mkdirs()
+
+        val css = javaClass<Html2CompilerPlugin>().getClassLoader()!!.getResourceAsStream(
+                "org/jetbrains/kotlin/doc/highlighter2/hightlight.css")!!
+
+        File(srcOutputRoot, "highlight.css").write { outputStream ->
+            css.copyTo(outputStream)
+            #()
+        }
+
+        for (file in context.getFiles().requireNoNulls()) {
+            processFile(file)
+        }
+    }
+
+    private fun processFile(psiFile: PsiFile) {
+        val relativePath = fileToWrite(psiFile)
+        val htmlFile = File(srcOutputRoot, relativePath.replaceFirst("\\.kt$", "") + ".html")
+
+        println("Generating $htmlFile")
+        htmlFile.getParentFile()!!.mkdirs()
+
+        // see http://maven.apache.org/jxr/xref/index.html
+
+        val template = object : HtmlTemplate() {
+            override fun render() {
+                html {
+                    head {
+                        title("${psiFile.getName()} xref")
+                        linkCssStylesheet("highlight.css")
+                    }
+                    body {
+                        table(style = "white-space: pre; font-size: 10pt; font-family: Monaco, monospace") {
+                            tr {
+                                td(style = "margin-right: 1.5em; text-align: right") {
+                                    val text = psiFile.getText()!!
+                                    val lineCount =
+                                        text.count { c -> c == '\n' } + (if (text.endsWith('\n')) 0 else 1)
+
+                                    for (i in 1..lineCount) {
+                                        val label = "$i"
+                                        a(name = "$label", href = "#$label") {
+                                            print(i)
+                                        }
+                                        println()
+                                    }
+                                }
+
+                                td {
+                                    fun classForPsi(psi: PsiElement): String? = when (psi) {
+                                        is LeafPsiElement -> {
+                                            val elementType = psi.getElementType()
+                                            if (elementType == JetTokens.WHITE_SPACE)
+                                                null
+                                            else if (elementType == JetTokens.DOC_COMMENT)
+                                                "hl-comment"
+                                            else if (JetTokens.KEYWORDS.contains(elementType))
+                                                "hl-keyword"
+                                            else
+                                                // TODO
+                                                elementType.toString()
+                                        }
+                                        // TODO
+                                        else -> psi.getClass()!!.getName()
+                                    }
+
+                                    for (t in splitPsi(psiFile)) {
+                                        val text = t._1
+                                        val psi = t._2
+                                        span(className = classForPsi(psi)) {
+                                            print(text.escapeHtml())
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        template.renderTo(htmlFile)
+    }
+
+}

libraries/tools/kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/highlighter2/hightlight.css

@@ -0,0 +1,7 @@
+.hl-comment {
+    color: #808080;
+}
+
+.hl-keyword {
+    color: #000080;
+}

libraries/tools/kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/highlighter2/ioUtils.kt

@@ -0,0 +1,8 @@
+package org.jetbrains.kotlin.doc.highlighter2
+
+import java.io.File
+import java.io.FileInputStream
+import java.io.FileOutputStream
+
+fun File.write(callback: (FileOutputStream) -> Unit) =
+    FileOutputStream(this).use<FileOutputStream, Unit>(callback)

libraries/tools/kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/highlighter2/psiUtils.kt

@@ -0,0 +1,41 @@
+package org.jetbrains.kotlin.doc.highlighter2
+
+import org.jetbrains.jet.internal.com.intellij.psi.PsiElement
+import java.util.List
+
+private fun PsiElement.getTextChildRelativeOffset() =
+    getTextRange()!!.getStartOffset() - getParent()!!.getTextRange()!!.getStartOffset()
+
+private fun PsiElement.getAllChildren(): List<PsiElement> {
+    val r = listBuilder<PsiElement>()
+    var child = getFirstChild()
+    while (child != null) {
+        r.add(child!!)
+        child = child!!.getNextSibling()
+    }
+    return r.build()
+}
+
+private fun splitPsiImpl(psi: PsiElement, listBuilder: ImmutableArrayListBuilder<#(String, PsiElement)>) {
+    var lastPos = 0
+    for (child in psi.getAllChildren()) {
+        if (child.getTextChildRelativeOffset() > lastPos) {
+            val text = psi.getText()!!.substring(lastPos, child.getTextChildRelativeOffset())
+            listBuilder.add(#(text, psi))
+        }
+        splitPsiImpl(child, listBuilder)
+        lastPos = child.getTextChildRelativeOffset() + child.getTextRange()!!.getLength()
+    }
+    if (lastPos < psi.getTextRange()!!.getLength()) {
+        val text = psi.getText()!!.substring(lastPos)
+        listBuilder.add(#(text, psi))
+    }
+}
+
+fun splitPsi(psi: PsiElement): List<#(String, PsiElement)> {
+    val listBuilder = listBuilder<#(String, PsiElement)>()
+    splitPsiImpl(psi, listBuilder)
+    return listBuilder.build()
+}
+
+

libraries/tools/kdoc/src/main/kotlin/org/jetbrains/kotlin/template/HtmlTemplate.kt

@@ -0,0 +1,91 @@
+package org.jetbrains.kotlin.template
+
+import java.util.List
+
+
+abstract class HtmlTemplate() : TextTemplate() {
+
+    fun tag(
+            tagName: String,
+            style: String? = null,
+            className: String? = null,
+            attributes: List<#(String, String)> = arrayList(),
+            content: () -> Unit) {
+        val allAttributesBuilder = listBuilder<#(String, String)>()
+        if (style != null)
+            allAttributesBuilder.add(#("style", style))
+        if (className != null)
+            allAttributesBuilder.add(#("class", className))
+
+        // TODO: add addAll to ListBuilder
+        for (attribute in attributes)
+            allAttributesBuilder.add(attribute)
+
+        val allAttributes = allAttributesBuilder.build()
+
+        print(
+                if (allAttributes.isEmpty()) {
+                    "<$tagName>"
+                }
+                else {
+                    // TODO: escape values
+                    "<$tagName ${allAttributes.map { t -> "${t._1}='${t._2.escapeHtml()}'" }.makeString(" ")}>"
+                }
+        )
+        content()
+        print("</$tagName>")
+    }
+
+    fun text(text: String) {
+        print(text.escapeHtml())
+    }
+
+    fun tag(tagName: String, content: String) =
+    tag(tagName) {
+        text(content)
+    }
+
+    fun html(content: () -> Unit) {
+        println("<!DOCTYPE html>")
+        tag(tagName = "html", content = content)
+    }
+
+    fun head(content: () -> Unit) =
+        tag(tagName = "head", content = content)
+
+    fun linkCssStylesheet(href: String) =
+        tag(
+            tagName = "link",
+            attributes = arrayList(#("rel", "stylesheet"), #("type", "text/css"), #("href", href))) {}
+
+    fun body(style: String? = null, className: String? = null, content: () -> Unit) =
+        tag(tagName = "body", style = style, className = className, content = content)
+
+    fun table(style: String? = null, className: String? = null, content: () -> Unit) =
+        tag(tagName = "table", style = style, className = className, content = content)
+
+    fun tr(style: String? = null, className: String? = null, content: () -> Unit) =
+        tag(tagName = "tr", style = style, className = className, content = content)
+
+    fun td(style: String? = null, className: String? = null, content: () -> Unit) =
+        tag(tagName = "td", style = style, className = className, content = content)
+
+    fun title(title: String) =
+        tag("title", title)
+
+    fun div(style: String? = null, className: String? = null, content: () -> Unit) =
+        tag(tagName = "div", style = style, className = className, content = content)
+
+    fun span(style: String? = null, className: String? = null, content: () -> Unit) =
+        tag(tagName = "span", style = style, className = className, content = content)
+
+    fun a(href: String? = null, name: String? = null, content: () -> Unit) {
+        val attributes = listBuilder<#(String, String)>()
+        if (href != null)
+            attributes.add(#("href", href))
+        if (name != null)
+            attributes.add(#("name", name))
+        tag(tagName = "a", attributes = attributes.build(), content = content)
+    }
+}
+

libraries/tools/kdoc/src/main/kotlin/org/jetbrains/kotlin/template/TemplateCore.kt

@@ -7,6 +7,8 @@ import java.io.OutputStream
 import java.io.OutputStreamWriter
 import java.io.FileWriter
 import java.io.File
+import java.util.List
+import kotlin.dom.attribute
 
 /**
  * Represents a generic API to templates which should be usable
@@ -29,14 +31,14 @@ trait Template {
  * We abstract away java.io.* APIs here to make the JS implementation simpler
  */
 trait Printer {
-    fun print(value: Any): Unit
+    fun print(value: Any?): Unit
     //fun print(text: String): Unit
 
 }
 
 
 class NullPrinter() : Printer {
-    override fun print(value: Any) {
+    override fun print(value: Any?) {
         throw UnsupportedOperationException("No Printer defined on the Template")
     }
 }
@@ -61,13 +63,15 @@ abstract class TextTemplate() : TemplateSupport(), Printer {
         printer.print(this)
     }
 
-    override fun print(value: Any) = printer.print(value)
+    override fun print(value: Any?) = printer.print(value)
 
-    fun println(value: Any) {
+    fun println(value: Any?) {
         print(value)
         print(newline)
     }
 
+    fun println() = println("")
+
     fun renderToText(): String {
         val buffer = StringWriter()
         renderTo(buffer)
@@ -97,12 +101,13 @@ abstract class TextTemplate() : TemplateSupport(), Printer {
 }
 
 
+
 /**
  * A Printer implementation which uses a Writer
  */
 class WriterPrinter(val writer: Writer) : Printer {
 
-    override fun print(value: Any) {
+    override fun print(value: Any?) {
         // TODO should be using a formatter to do the conversion
         writer.write(value.toString())
     }

libraries/tools/kdoc/src/main/kotlin/org/jetbrains/kotlin/template/stringUtils.kt

@@ -0,0 +1,8 @@
+package org.jetbrains.kotlin.template
+
+fun String.escapeHtml() =
+    this
+        .replace("&", "&")
+        .replace("\"", """)
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")

libraries/tools/kdoc/src/test/kotlin/test/kotlin/psiUtilsTest.kt

@@ -0,0 +1,58 @@
+package test.kotlin
+
+import org.jetbrains.jet.cli.jvm.compiler.JetCoreEnvironment
+import org.junit.Before
+import org.jetbrains.jet.internal.com.intellij.openapi.Disposable
+import org.jetbrains.jet.lang.resolve.java.CompilerDependencies
+import org.jetbrains.jet.lang.resolve.java.CompilerSpecialMode
+import org.junit.After
+import org.jetbrains.jet.internal.com.intellij.openapi.util.Disposer
+import org.junit.Test
+import org.jetbrains.jet.plugin.JetLanguage
+import org.jetbrains.jet.internal.com.intellij.testFramework.LightVirtualFile
+import org.jetbrains.jet.internal.com.intellij.openapi.vfs.CharsetToolkit
+import org.jetbrains.jet.internal.com.intellij.psi.PsiFile
+import org.jetbrains.jet.lang.psi.JetFile
+import org.jetbrains.jet.internal.com.intellij.psi.PsiFileFactory
+import org.jetbrains.jet.internal.com.intellij.psi.impl.PsiFileFactoryImpl
+import org.jetbrains.kotlin.doc.highlighter2.splitPsi
+import org.junit.Assert
+import java.util.List
+
+class PsiUtilsTest {
+
+    val rootDisposable = object : Disposable {
+        public override fun dispose() {
+        }
+    }
+
+    private var environment: JetCoreEnvironment? = null
+
+    [Before]
+    fun before() {
+        System.setProperty("java.awt.headless", "true")
+
+        val compilerDependencies = CompilerDependencies.compilerDependenciesForProduction(CompilerSpecialMode.JDK_HEADERS)
+        environment = JetCoreEnvironment(rootDisposable, compilerDependencies)
+    }
+
+    [After]
+    fun after() {
+        Disposer.dispose(rootDisposable)
+    }
+
+    private fun createFile(content: String): JetFile {
+        val virtualFile = LightVirtualFile("file.kt", JetLanguage.INSTANCE, content);
+        virtualFile.setCharset(CharsetToolkit.UTF8_CHARSET);
+        return (PsiFileFactory.getInstance(environment!!.getProject()) as PsiFileFactoryImpl)
+                .trySetupPsiForFile(virtualFile, JetLanguage.INSTANCE, true, false) as JetFile
+    }
+
+    [Test]
+    fun splitPsi() {
+        val file = createFile("class Foo")
+        val items: List<String> = splitPsi(file).map { t -> t._1 }
+        Assert.assertEquals(arrayList("class", " ", "Foo"), items)
+    }
+
+}