add syntax highlighting to included code in kdocs #KT-1660 Fixed

libraries/kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/highlighter/SyntaxHighlighter.kt

@@ -0,0 +1,138 @@
+package org.jetbrains.kotlin.doc.highlighter
+
+import com.intellij.psi.*
+import com.intellij.psi.tree.IElementType
+import com.intellij.psi.tree.TokenSet
+import java.util.HashMap
+import java.util.Map
+import kotlin.template.HtmlFormatter
+import org.jetbrains.jet.lexer.*
+
+fun main(args: Array<String>) {
+    val tool = SyntaxHighligher()
+    val answer = tool.highlight("""    val x = arrayList(1, 2, 3)
+    println("hello")""")
+    println(answer)
+}
+
+/**
+ * Syntax highlights Kotlin code
+ */
+class SyntaxHighligher() {
+    var formatter: HtmlFormatter = HtmlFormatter()
+    val styleMap = createStyleMap()
+
+    /** Highlights the given kotlin code as HTML */
+    fun highlight(val code: String): String {
+        try {
+            val builder = StringBuilder()
+            builder.append(
+            "<div class=\"code panel\" style=\"border-width: 1px\">" +
+            "<div class=\"codeContent panelContent\">" +
+            "<div class=\"container\">"
+            )
+
+            // lets add the leading whitespace first
+            var idx = 0
+            while (Character.isWhitespace(code[idx])) {
+                idx++
+            }
+            if (idx > 0) {
+                val space = code.substring(0, idx)
+                builder.append("""<code class="jet whitespace">$space</code>""")
+            }
+            val lexer = JetLexer()
+            lexer.start(code)
+            val end = lexer.getTokenEnd()
+            while (true) {
+                lexer.advance()
+                val token = lexer.getTokenType()
+                if (token == null) break
+                val tokenText = lexer.getTokenSequence().toString().replaceAll("\n", "\r\n")
+                var style: String? = null
+                if (token is JetKeywordToken) {
+                    style = "keyword"
+                } else if (token == JetTokens.IDENTIFIER) {
+                    val types = JetTokens.SOFT_KEYWORDS?.getTypes()
+                    if (types != null) {
+                        for (softKeyword in types) {
+                            if (softKeyword is JetKeywordToken) {
+                                if (softKeyword.getValue().equals(tokenText)) {
+                                    style = "softkeyword"
+                                    break
+                                }
+                            }
+                        }
+                    }
+                    style = if (style == null) "plain" else style
+                } else if (styleMap.containsKey(token)) {
+                    style = styleMap.get(token)
+                    if (style == null) {
+                        println("Warning: No style for token $token")
+                    }
+                } else {
+                    style = "plain"
+                }
+                builder.append("""<code class="jet $style">""")
+                formatter.format(builder, tokenText)
+                builder.append("</code>")
+            }
+
+            builder.append("</div>")
+            builder.append("</div>")
+            builder.append("</div>")
+            return builder.toString() ?: ""
+        } catch (e: Exception) {
+            println("Warning: failed to parse code $e")
+            val builder = StringBuilder()
+            builder.append("""<div class="jet herror">Jet highlighter error ["${e.javaClass.getSimpleName()}"]: """)
+            formatter.format(builder, e.getMessage())
+            builder.append("<br/>")
+            builder.append("Original text:")
+            builder.append("<pre>")
+            formatter.format(builder, code)
+            builder.append("</pre>")
+            builder.append("</div>")
+            return builder.toString() ?: ""
+        }
+    }
+
+
+    protected fun createStyleMap(): Map<IElementType?, String> {
+        val styleMap = HashMap<IElementType?, String>()
+
+        fun putAll(tokenSet: TokenSet?, style: String): Unit {
+            if (tokenSet != null) {
+                for (token in tokenSet.getTypes().orEmpty()) {
+                    styleMap.put(token, style)
+                }
+            }
+        }
+
+        styleMap.put(JetTokens.BLOCK_COMMENT, "jet-comment")
+        styleMap.put(JetTokens.DOC_COMMENT, "jet-comment")
+        styleMap.put(JetTokens.EOL_COMMENT, "jet-comment")
+        styleMap.put(JetTokens.WHITE_SPACE, "whitespace")
+        styleMap.put(JetTokens.INTEGER_LITERAL, "number")
+        styleMap.put(JetTokens.FLOAT_LITERAL, "number")
+        styleMap.put(JetTokens.OPEN_QUOTE, "string")
+        styleMap.put(JetTokens.REGULAR_STRING_PART, "string")
+        styleMap.put(JetTokens.ESCAPE_SEQUENCE, "escape")
+        styleMap.put(JetTokens.LONG_TEMPLATE_ENTRY_START, "escape")
+        styleMap.put(JetTokens.LONG_TEMPLATE_ENTRY_END, "escape")
+        styleMap.put(JetTokens.SHORT_TEMPLATE_ENTRY_START, "escape")
+        styleMap.put(JetTokens.ESCAPE_SEQUENCE, "escape")
+        styleMap.put(JetTokens.CLOSING_QUOTE, "string")
+        styleMap.put(JetTokens.CHARACTER_LITERAL, "string")
+        styleMap.put(JetTokens.LABEL_IDENTIFIER, "label")
+        styleMap.put(JetTokens.ATAT, "label")
+        styleMap.put(JetTokens.FIELD_IDENTIFIER, "field")
+        styleMap.put(TokenType.BAD_CHARACTER, "bad")
+        putAll(JetTokens.STRINGS, "string")
+        putAll(JetTokens.MODIFIER_KEYWORDS, "softkeyword")
+        putAll(JetTokens.SOFT_KEYWORDS, "softkeyword")
+        putAll(JetTokens.COMMENTS, "jet-comment")
+        putAll(JetTokens.OPERATIONS, "operation")
+        return styleMap
+    }
+}

libraries/kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/model/KotlinModel.kt

@@ -4,6 +4,7 @@ import kotlin.*
 import kotlin.util.*
 
 import org.jetbrains.kotlin.doc.KDocConfig
+import org.jetbrains.kotlin.doc.highlighter.SyntaxHighligher
 
 import java.util.*
 import org.jetbrains.jet.lang.descriptors.ClassDescriptor
@@ -180,6 +181,7 @@ class KModel(var context: BindingContext, val config: KDocConfig) {
     get() = packages.flatMap{ it.classes }
 
     public var markdownProcessor: PegDownProcessor = PegDownProcessor(Extensions.ALL)
+    public var highlighter: SyntaxHighligher = SyntaxHighligher()
 
     public val title: String
     get() = config.title
@@ -399,8 +401,7 @@ class KModel(var context: BindingContext, val config: KDocConfig) {
                                 val fnName = words[1].sure()
                                 val content = findFunctionInclude(psiElement, includeFile, fnName)
                                 if (content != null) {
-                                    // TODO ideally we'd use pygmentize or somehting here to format the Kotlin code...
+                                    text = highlighter.highlight(content)
-                                    text = "<pre><code>" + content + "</code></pre>\n"
                                 } else {
                                     warning("could not find function $fnName in file $includeFile from source file ${psiElement.getContainingFile()}")
                                 }

libraries/kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/templates/ClassTemplate.kt

@@ -28,7 +28,7 @@ ${pageTitle()}
 
 <META NAME="date" CONTENT="2012-01-09">
 <META NAME="date" CONTENT="2012-01-09">
-<LINK REL="stylesheet" TYPE="text/css" HREF="${relativePrefix()}stylesheet.css" TITLE="Style">
+${stylesheets()}
 
 <SCRIPT type="text/javascript">
 function windowTitle()

libraries/kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/templates/PackageFrameTemplate.kt

@@ -25,7 +25,7 @@ ${pkg.name} (${model.title})
 </TITLE>
 
 <META NAME="date" CONTENT="2012-01-09">
-<LINK REL ="stylesheet" TYPE="text/css" HREF="${pkg.nameAsRelativePath}stylesheet.css" TITLE="Style">
+${stylesheets()}
 
 </HEAD>
 

libraries/kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/templates/PackageSummaryTemplate.kt

@@ -26,7 +26,7 @@ ${pkg.name} (${model.title})
 
 <META NAME="date" CONTENT="2012-01-09">
 <META NAME="date" CONTENT="2012-01-09">
-<LINK REL="stylesheet" TYPE="text/css" HREF="${pkg.nameAsRelativePath}stylesheet.css" TITLE="Style">
+${stylesheets()}
 
 <SCRIPT type="text/javascript">
 function windowTitle()

libraries/kdoc/src/main/kotlin/org/jetbrains/kotlin/doc/templates/PackageTemplateSupport.kt

@@ -251,4 +251,9 @@ abstract class PackageTemplateSupport(open val pkg: KPackage) : KDocTemplate() {
             println(link(a))
         }
     }
+
+    fun stylesheets(): String {
+        return """<LINK REL="stylesheet" TYPE="text/css" HREF="${relativePrefix()}stylesheet.css" TITLE="Style">
+        <LINK REL="stylesheet" TYPE="text/css" HREF="${relativePrefix()}kotlin.css" TITLE="Style">"""
+    }
 }

libraries/website/src/main/apidocs/kotlin.css

@@ -0,0 +1,74 @@
+.jet {
+  white-space: pre;
+  display: inline;
+}
+
+.jet.keyword {
+  font-weight: bold;
+  color: #369;
+}
+
+.jet.softkeyword {
+  color: #369;
+}
+
+.jet.jet-comment {
+  color : green;
+
+}
+
+.jet.number {
+  color : blue;
+}
+.jet.string {
+  color : #000099;
+}
+.jet.escape {
+  font-weight: bold;
+  color : black;
+}
+.jet.label {
+  color : blue;
+}
+.jet.field {
+  color : blue;
+  font-weight: bold;
+}
+.jet.operation {
+  color : gray;
+  font-weight: bold;
+}
+.jet.bad {
+  background-color : #FF9999;
+  color:black;
+}
+.jet.error {
+  background: url(resources/underline.gif) bottom repeat-x;
+  display:inline;
+}
+.jet.warning {
+  background-color : #f5eabb;
+  display:inline;
+}
+.jet.unresolved {
+  color : red;
+  display:inline;
+}
+
+.jet.herror {
+  background-color: #FFAAAA;
+}
+.jet.hwarning {
+  background-color: #FFAAAA;
+  display:inline;
+}
+
+.jet.ref:link    { color: black; }
+.jet.ref:visited { color: black; }
+.jet.ref:hover   { color:blue; text-decoration:underline; }
+.jet.ref:active  { color:blue; text-decoration:underline; }
+
+.jet.anchor:link    { }
+.jet.anchor:visited { }
+.jet.anchor:hover   { color:blue; text-decoration:underline; }
+.jet.anchor:active  { color:blue; text-decoration:underline; }