Rendering the grammar for Confluence

.idea/ant.xml

@@ -10,6 +10,14 @@
       <maximumStackSize value="2" />
       <properties />
     </buildFile>
+    <buildFile url="file://$PROJECT_DIR$/grammar/buildGrammarLexer.xml">
+      <additionalClassPath />
+      <antReference projectDefault="true" />
+      <customJdkName value="" />
+      <maximumHeapSize value="128" />
+      <maximumStackSize value="2" />
+      <properties />
+    </buildFile>
   </component>
 </project>
 

grammar/buildGrammarLexer.xml

@@ -0,0 +1,43 @@
+<project name="GrammarLexer" default="lexer">
+    <property name="home" value="${basedir}"/>
+    <property file="${home}/../idea/idea.properties"/>
+
+    <property name="flex.base" value="${idea.home}/tools/lexer/jflex-1.4"/>
+    <property name="out.dir" value="${basedir}/tmpout"/>
+
+    <macrodef name="flex">
+      <attribute name="flexfile"/>
+      <attribute name="destdir"/>
+      <attribute name="skeleton" default="${idea.home}/tools/lexer/idea-flex.skeleton"/>
+      <sequential>
+        <delete dir="${out.dir}"/>
+        <mkdir dir="${out.dir}"/>
+        <java classname="JFlex.Main"
+              jvmargs="-Xmx512M"
+              fork="true"
+              failonerror="true">
+          <arg value="-sliceandcharat"/>
+          <arg value="-skel"/>
+          <arg value="@{skeleton}"/>
+          <arg value="-d"/>
+          <arg value="${out.dir}"/>
+          <arg value="@{flexfile}"/>
+          <classpath>
+            <pathelement location="${flex.base}/lib/JFlex.jar"/>
+          </classpath>
+        </java>
+        <move todir="@{destdir}">
+          <fileset dir="${out.dir}">
+            <include name="*.java"/>
+          </fileset>
+        </move>
+        <delete dir="${out.dir}"/>
+      </sequential>
+    </macrodef>
+
+    <target name="lexer">
+        <echo message="${flex.base}"/>
+        <flex flexfile="${home}/src/org/jetbrains/jet/grammar/Grammar.flex"
+              destdir="${home}//src/org/jetbrains/jet/grammar/"/>
+    </target>
+</project>

grammar/grammar.iml

@@ -7,6 +7,15 @@
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="module-library">
+      <library>
+        <CLASSES>
+          <root url="jar://$MODULE_DIR$/../../idea/idea.app/lib/util.jar!/" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES />
+      </library>
+    </orderEntry>
   </component>
 </module>
 

grammar/src/attributes.grm

@@ -1,3 +1,7 @@
+/**
+h2. Annotations
+*/
+
 attributes
   : attributeAnnotation*
   ;

grammar/src/class.grm

@@ -1,3 +1,7 @@
+/**
+h2. Classes
+*/
+
 /*
 
 internal class Example<X, T : Comparable<X>>(protected val x : Foo<X, T>, y : Some)

grammar/src/class_members.grm

@@ -1,3 +1,7 @@
+/**
+h3. Class members
+*/
+
 /*
 class Example(a : Foo, i : Int) : Bar(i), Some {
   // constrtuctors:
@@ -37,20 +41,24 @@ memberDeclaration
   : function
   : property
   : class
-  : extension
+  : extension                                                    §
   : typedef
   : anonymousInitializer
   ;
 
+anonymousInitializer
+  : block
+  ;
+
 classObject
   : modifiers "class" object
   ;
 
 constructor
-  : modifiers "this" functionParameters (":" initializer{","}) block?
+  : modifiers "this" valueParameters (":" initializer{","}) block?
   ;
 
-functionParameters
+valueParameters
   : "(" functionParameter{","}? ")" // default values
   ;
 
@@ -71,7 +79,7 @@ function
   : modifiers "fun" typeParameters?
       (type "." | attributes/*for receiver type*/)?
       SimpleName
-      typeParameters? functionParameters (":" type)?
+      typeParameters? valueParameters (":" type)?
       functionBody?
   ;
 

grammar/src/control.grm

@@ -1,3 +1,7 @@
+/**
+h3. Control structures
+*/
+
 if
   : "if" "(" expression ")" expression SEMI? ("else" expression)?
   ;
@@ -21,7 +25,7 @@ loop
   ;
 
 for
-  : "for" "(" attributes valOrVar? SimpleName (":" type)? "in" expression ")" expression
+  : "for" "(" attributes ("val" | "var")? SimpleName (":" type)? "in" expression ")" expression
   ;
 
 while

grammar/src/enum.grm

@@ -1,7 +1,11 @@
+/**
+h3. Enum classes
+*/
+
 enumClassBody
   : "{" enumEntry* "}"
   ;
 
 enumEntry
-  : modifiers SimpleName typeParameters? primaryConstructorParameters? (":" initializer{","})? classBody?
+  : modifiers SimpleName typeParameters? valueParameters? (":" initializer{","})? classBody?
   ;

grammar/src/expressions.grm

@@ -1,3 +1,7 @@
+/**
+h2. Expressions
+*/
+
 /*
 Decreasing precedence:
   memberAccessOperation
@@ -108,7 +112,7 @@ literalConstant
   : StringWithTemplates
   : NoEscapeString
   : IntegerLiteral
-  : LongLiteral
+  : HexadecimalLiteral
   : CharacterLiteral
   : FloatLiteral
   : "null"
@@ -148,7 +152,7 @@ typeOperation
   : "as" : "as?" : ":"
   ;
 
-isOPeration
+isOperation
   : "is" : "!is"
   ;
 
@@ -210,17 +214,14 @@ jump
 // yield ?
   ;
 
-stringLiteral
-  : StringWithTemplates
-  : NoEscapeString
-  ;
-
-tupleLiteral // Ambiguity when after a SimpleName (infix call). In this case (e) is treated as an expression in parentheses
-             // to put a tuple, write write ((e))
+// Ambiguity when after a SimpleName (infix call). In this case (e) is treated as an expression in parentheses
+// to put a tuple, write write ((e))
+tupleLiteral
   : "(" ((SimpleName "=")? expression){","} ")"
   ;
 
-functionLiteral  // one can use "it" as a parameter name
+// one can use "it" as a parameter name
+functionLiteral
   : "{" statements "}"
   : "{" (type ".")? modifiers SimpleName "=>" statements "}"
   : "{" (type ".")? "(" (modifiers SimpleName (":" type)?){","} ")" (":" type)? "=>" statements "}"

grammar/src/lexical.grm

@@ -1,15 +1,42 @@
-SimpleName : /*java identifier*/;
-FieldName : "$" SimpleName;
-LabelName : "@" SimpleName;
-[helper] Digit : ["0".."9"];
-IntegerLiteral : Digit+ "L"?
-[helper] HexDigit : Digit | ["A"-"F", "a"-"f"];
-HexadecimalLiteral : "0x" HexDigit+;
+/**
+h1. Lexical structure
+*/
 
-CharacterLiteral : /*character as in Java*/;
+SimpleName
+  : <java identifier>;
+FieldName
+  : "$" SimpleName;
+LabelName
+  : "@" SimpleName;
 
-StringWithTemplates : /*single-quoted string, $ can be escaped*/;
-NoEscapeString : /* """-quoted string */;
+[helper]
+Digit
+  : ["0".."9"];
+
+IntegerLiteral
+  : Digit+ "L"?
+
+FloatLiteral
+  : <java double literal>;
+
+[helper]
+HexDigit
+  : Digit | ["A"-"F", "a"-"f"];
+
+HexadecimalLiteral
+  : "0x" HexDigit+;
+
+CharacterLiteral
+  : <character as in Java>;
+
+StringWithTemplates
+  : <single-quoted string, $ can be escaped>;
+
+NoEscapeString
+  : <"""-quoted string>;
+
+SEMI
+  : <semicolon or newline>;
 
 /* Symbols:
 

grammar/src/modifiers.grm

@@ -1,3 +1,7 @@
+/**
+h3. Modifiers
+*/
+
 modifiers
   : modifier*
   ;

grammar/src/notation.grm

@@ -0,0 +1,21 @@
+/**
+h1. Notation
+
+This section informally explains the grammar notation used below.
+
+h2. Symbols and naming
+
+_Terminal symbol_ names start with an uppercase letter, e.g. *SimpleName*
+_Nonterminal symbol_ names start with lowercase letter, e.g. *jetFile*
+Each _production_ starts with a colon (*:*)
+_Symbol definitions_ may have many productions and are terminated by a semicolon (*;*)
+Symbol definitions may be prepended with _attributes_, e.g. {{start}} attribute denotes a start symbol
+
+h2. EBNF expressions
+
+Operator {color:blue}*|*{color} denotes _alternative_
+Operator {color:blue}*\**{color} denotes _iteration_ (zero or more)
+Operator {color:blue}*+*{color} denotes _iteration_ (one or one)
+Operator {color:blue}*?*{color} denotes _option_ (zero or one)
+alpha{color:blue}*{*{color}beta{color:blue}*}*{color} denotes a nonempty _beta_-separated list of _alpha_s
+*/

grammar/src/org/jetbrains/jet/grammar/Annotation.java

@@ -0,0 +1,15 @@
+package org.jetbrains.jet.grammar;
+
+/**
+ * @author abreslav
+ */
+public class Annotation extends Token {
+    public Annotation(CharSequence text) {
+        super(text);
+    }
+
+    @Override
+    public String toString() {
+        return "{{" + super.toString().replaceAll("\\[", "").replaceAll("\\]", "") + "}}";
+    }
+}

grammar/src/org/jetbrains/jet/grammar/Comment.java

@@ -0,0 +1,15 @@
+package org.jetbrains.jet.grammar;
+
+/**
+ * @author abreslav
+ */
+public class Comment extends Token {
+    public Comment(CharSequence text) {
+        super(text);
+    }
+
+    @Override
+    public String toString() {
+        return "";//getText().toString().replaceAll("\\{", "\\\\{").replaceAll("\\[", "\\\\[").replaceAll("\\(", "\\\\(");
+    }
+}

grammar/src/org/jetbrains/jet/grammar/ConfluenceHyperlinksGenerator.java

@@ -0,0 +1,150 @@
+package org.jetbrains.jet.grammar;
+
+import java.awt.*;
+import java.awt.datatransfer.Clipboard;
+import java.awt.datatransfer.ClipboardOwner;
+import java.awt.datatransfer.StringSelection;
+import java.awt.datatransfer.Transferable;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.*;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author abreslav
+ */
+public class ConfluenceHyperlinksGenerator {
+
+    public static void main(String[] args) throws IOException {
+        String grammarExtension = "grm";
+        File grammarDir = new File("grammar/src");
+
+        List<String> fileNamesInOrder = Arrays.asList(
+                "notation",
+                "lexical",
+                "toplevel",
+                "class",
+                "class_members",
+                "enum",
+                "types",
+                "modifiers",
+                "attributes",
+                "control",
+                "expressions",
+                "when"
+        );
+
+        StringBuilder allRules = new StringBuilder();
+        Set<File> used = new HashSet<File>();
+        for (String fileName : fileNamesInOrder) {
+            File file = new File(grammarDir, fileName + "." + grammarExtension);
+            used.add(file);
+            FileReader fileReader = new FileReader(file);
+            try {
+                int c;
+                while ((c = fileReader.read()) != -1) {
+                    allRules.append((char) c);
+                }
+                allRules.append("\n");
+                allRules.append("\n");
+            } finally {
+                fileReader.close();
+            }
+        }
+
+        for (File file : grammarDir.listFiles()) {
+            if (file.getName().endsWith(grammarExtension)) {
+                if (!used.contains(file)) {
+                    throw new IllegalStateException("Unused grammar file : " + file.getAbsolutePath());
+                }
+            }
+        }
+
+        StringBuilder output = new StringBuilder();
+
+        Pattern symbolReference = Pattern.compile("^\\w+$", Pattern.MULTILINE);
+        Matcher matcher = symbolReference.matcher(allRules);
+        int copiedUntil = 0;
+        while (matcher.find()) {
+            int start = matcher.start();
+            output.append(allRules.subSequence(copiedUntil, start));
+
+            String group = matcher.group();
+            output.append("&").append(group);
+            copiedUntil = matcher.end();
+        }
+        output.append(allRules.subSequence(copiedUntil, allRules.length()));
+//        System.out.println(output);
+
+        _GrammarLexer grammarLexer = new _GrammarLexer((Reader) null);
+        grammarLexer
+                .reset(output, 0, output.length(), 0);
+
+        StringBuilder result = new StringBuilder("h1. Contents\n").append("{toc:style=disc|indent=20px}");
+
+        Set<String> declaredSymbols = new HashSet<String>();
+        Set<String> usedSymbols = new HashSet<String>();
+        List<Token> tokens = new ArrayList<Token>();
+
+        while (true) {
+            Token advance = grammarLexer.advance();
+            if (advance == null) {
+                break;
+            }
+            if (advance instanceof Declaration) {
+                Declaration declaration = (Declaration) advance;
+                declaredSymbols.add(declaration.getName());
+            }
+            else if (advance instanceof Identifier) {
+                Identifier identifier = (Identifier) advance;
+                usedSymbols.add(identifier.getName());
+            }
+            tokens.add(advance);
+        }
+
+        for (Token token : tokens) {
+            if (token instanceof Declaration) {
+                Declaration declaration = (Declaration) token;
+                if (!usedSymbols.contains(declaration.getName())) {
+                    result.append("(!) *Unused!* ");
+                    System.out.println("Unused: " + token);
+                }
+            }
+            else if (token instanceof Identifier) {
+                Identifier identifier = (Identifier) token;
+                if (!declaredSymbols.contains(identifier.getName())) {
+                    result.append("(!) *Undeclared!* ");
+                    System.out.println("Undeclared: " + token);
+                }
+            }
+            result.append(token);
+        }
+
+        Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
+        clipboard.setContents(new StringSelection(result.toString()), new ClipboardOwner() {
+            @Override
+            public void lostOwnership(Clipboard clipboard, Transferable contents) {
+
+            }
+        });
+    }
+
+    private static Set<Integer> recordStarts(StringBuilder allRules, String regex) {
+        Pattern symbolDeclaration = Pattern.compile(regex, Pattern.MULTILINE);
+        Matcher matcher = symbolDeclaration.matcher(allRules);
+        Set<Integer> declarationStarts = new HashSet<Integer>();
+        while (matcher.find()) {
+            if (matcher.groupCount() > 0) {
+                declarationStarts.add(matcher.start(1));
+            }
+            else {
+                declarationStarts.add(matcher.start());
+            }
+        }
+        return declarationStarts;
+    }
+}

grammar/src/org/jetbrains/jet/grammar/Declaration.java

@@ -0,0 +1,22 @@
+package org.jetbrains.jet.grammar;
+
+/**
+ * @author abreslav
+ */
+public class Declaration extends Token {
+    private final String name;
+
+    public Declaration(CharSequence text) {
+        super(text);
+        name = text.toString().substring(1);
+    }
+
+    @Override
+    public String toString() {
+        return "{anchor:" + name + "}*" + name + "*";
+    }
+
+    public String getName() {
+        return name;
+    }
+}

grammar/src/org/jetbrains/jet/grammar/DocComment.java

@@ -0,0 +1,16 @@
+package org.jetbrains.jet.grammar;
+
+/**
+ * @author abreslav
+ */
+public class DocComment extends Token {
+    public DocComment(CharSequence text) {
+        super(text);
+    }
+
+    @Override
+    public String toString() {
+        String s = super.toString();
+        return s.substring(3, s.length() - 2);
+    }
+}

grammar/src/org/jetbrains/jet/grammar/Grammar.flex

@@ -0,0 +1,102 @@
+/* It's an automatically generated code. Do not modify it. */
+package org.jetbrains.jet.grammar;
+
+//import com.intellij.lexer.*;
+//import com.intellij.psi.*;
+//import com.intellij.psi.tree.IElementType;
+
+//import org.jetbrains.jet.lexer.JetTokens;
+
+%%
+
+%unicode
+%class _GrammarLexer
+%function advance
+%type Token
+%eof{  return;
+%eof}
+
+DIGIT=[0-9]
+HEX_DIGIT=[0-9A-Fa-f]
+WHITE_SPACE_CHAR=[\ \n\t\f]
+
+// TODO: prohibit '$' in identifiers?
+PLAIN_IDENTIFIER=[:jletter:] [:jletterdigit:]*
+// TODO: this one MUST allow everything accepted by the runtime
+// TODO: Replace backticks by one backslash in the begining
+ESCAPED_IDENTIFIER = `{PLAIN_IDENTIFIER}`
+IDENTIFIER = {PLAIN_IDENTIFIER}|{ESCAPED_IDENTIFIER}
+DECLARATION_IDENTIFIER = "&" {PLAIN_IDENTIFIER}
+FIELD_IDENTIFIER = \${IDENTIFIER}
+LABEL_IDENTIFIER = \@{IDENTIFIER}
+
+BLOCK_COMMENT=("/*"[^"*"]{COMMENT_TAIL})|"/*"
+// TODO: Wiki markup for doc comments?
+DOC_COMMENT="/*""*"+("/"|([^"/""*"]{COMMENT_TAIL}))?
+COMMENT_TAIL=([^"*"]*("*"+[^"*""/"])?)*("*"+"/")?
+EOL_COMMENT="/""/"[^\n]*
+
+INTEGER_LITERAL={DECIMAL_INTEGER_LITERAL}|{HEX_INTEGER_LITERAL}|{BIN_INTEGER_LITERAL}
+DECIMAL_INTEGER_LITERAL=(0|([1-9]({DIGIT})*))
+HEX_INTEGER_LITERAL=0[Xx]({HEX_DIGIT})*
+BIN_INTEGER_LITERAL=0[Bb]({DIGIT})*
+LONG_LITERAL=({INTEGER_LITERAL})[Ll]
+
+//FLOAT_LITERAL=(({FLOATING_POINT_LITERAL1})[Ff])|(({FLOATING_POINT_LITERAL2})[Ff])|(({FLOATING_POINT_LITERAL3})[Ff])|(({FLOATING_POINT_LITERAL4})[Ff])
+//DOUBLE_LITERAL=(({FLOATING_POINT_LITERAL1})[Dd]?)|(({FLOATING_POINT_LITERAL2})[Dd]?)|(({FLOATING_POINT_LITERAL3})[Dd]?)|(({FLOATING_POINT_LITERAL4})[Dd])
+DOUBLE_LITERAL={FLOATING_POINT_LITERAL1}|{FLOATING_POINT_LITERAL2}|{FLOATING_POINT_LITERAL3}|{FLOATING_POINT_LITERAL4}
+FLOATING_POINT_LITERAL1=({DIGIT})+"."({DIGIT})+({EXPONENT_PART})?
+FLOATING_POINT_LITERAL2="."({DIGIT})+({EXPONENT_PART})?
+FLOATING_POINT_LITERAL3=({DIGIT})+({EXPONENT_PART})
+FLOATING_POINT_LITERAL4=({DIGIT})+
+EXPONENT_PART=[Ee]["+""-"]?({DIGIT})*
+HEX_FLOAT_LITERAL={HEX_SIGNIFICAND}{BINARY_EXPONENT}[Ff]
+//HEX_DOUBLE_LITERAL={HEX_SIGNIFICAND}{BINARY_EXPONENT}[Dd]?
+HEX_DOUBLE_LITERAL={HEX_SIGNIFICAND}{BINARY_EXPONENT}?
+BINARY_EXPONENT=[Pp][+-]?{DIGIT}+
+HEX_SIGNIFICAND={HEX_INTEGER_LITERAL}|0[Xx]{HEX_DIGIT}*\.{HEX_DIGIT}+
+//HEX_SIGNIFICAND={HEX_INTEGER_LITERAL}|{HEX_INTEGER_LITERAL}\.|0[Xx]{HEX_DIGIT}*\.{HEX_DIGIT}+
+
+CHARACTER_LITERAL="'"([^\\\'\n]|{ESCAPE_SEQUENCE})*("'"|\\)?
+// TODO: introduce symbols (e.g. 'foo) as another way to write string literals
+STRING_LITERAL=\"([^\\\"\n]|{ESCAPE_SEQUENCE})*(\"|\\)?
+ANGLE_STRING_LITERAL=\<([^\\\>\n])*(\>|\\)?
+ESCAPE_SEQUENCE=\\[^\n]
+
+// ANY_ESCAPE_SEQUENCE = \\[^]
+THREE_QUO = (\"\"\")
+ONE_TWO_QUO = (\"[^\"]) | (\"\"[^\"])
+QUO_STRING_CHAR = [^\"] | {ONE_TWO_QUO}
+RAW_STRING_LITERAL = {THREE_QUO} {QUO_STRING_CHAR}* {THREE_QUO}?
+
+%%
+
+<YYINITIAL> {BLOCK_COMMENT} { return new Comment(yytext()); }
+<YYINITIAL> {DOC_COMMENT} { return new DocComment(yytext()); }
+
+<YYINITIAL> ({WHITE_SPACE_CHAR})+ { return new WhiteSpace(yytext()); }
+
+<YYINITIAL> {EOL_COMMENT} { return new Comment(yytext()); }
+
+<YYINITIAL> {STRING_LITERAL} { return new StringToken(yytext()); }
+<YYINITIAL> {ANGLE_STRING_LITERAL} { return new StringToken(yytext()); }
+<YYINITIAL> {IDENTIFIER} { return new Identifier(yytext()); }
+<YYINITIAL> "[" {IDENTIFIER} "]" { return new Annotation(yytext()); }
+<YYINITIAL> {DECLARATION_IDENTIFIER} { return new Declaration(yytext()); }
+
+<YYINITIAL> ":"          { return new SymbolToken(yytext()); }
+<YYINITIAL> "{"          { return new SymbolToken("\\" + yytext()); }
+<YYINITIAL> "}"          { return new SymbolToken("\\" + yytext()); }
+<YYINITIAL> "["          { return new SymbolToken("\\" + yytext()); }
+<YYINITIAL> "]"          { return new SymbolToken("\\" + yytext()); }
+<YYINITIAL> "("          { return new SymbolToken("\\" + yytext()); }
+<YYINITIAL> ")"          { return new SymbolToken("\\" + yytext()); }
+<YYINITIAL> "*"          { return new SymbolToken(yytext()); }
+<YYINITIAL> "+"          { return new SymbolToken(yytext()); }
+<YYINITIAL> "?"          { return new SymbolToken(yytext()); }
+<YYINITIAL> "|"          { return new SymbolToken(yytext()); }
+<YYINITIAL> "-"          { return new SymbolToken(yytext()); }
+<YYINITIAL> "."          { return new SymbolToken(yytext()); }
+
+<YYINITIAL> . { return new Other(yytext()); }
+

grammar/src/org/jetbrains/jet/grammar/Identifier.java

@@ -0,0 +1,22 @@
+package org.jetbrains.jet.grammar;
+
+/**
+ * @author abreslav
+ */
+public class Identifier extends Token {
+    private final String name;
+
+    public Identifier(CharSequence text) {
+        super(text);
+        name = text.toString();
+    }
+
+    @Override
+    public String toString() {
+        return "[#" + getText() + "]";
+    }
+
+    public String getName() {
+        return name;
+    }
+}

grammar/src/org/jetbrains/jet/grammar/Other.java

@@ -0,0 +1,10 @@
+package org.jetbrains.jet.grammar;
+
+/**
+ * @author abreslav
+ */
+public class Other extends Token {
+    public Other(CharSequence text) {
+        super(text);
+    }
+}

grammar/src/org/jetbrains/jet/grammar/StringToken.java

@@ -0,0 +1,15 @@
+package org.jetbrains.jet.grammar;
+
+/**
+ * @author abreslav
+ */
+public class StringToken extends Token {
+    public StringToken(CharSequence text) {
+        super(text);
+    }
+
+    @Override
+    public String toString() {
+        return "{color:green}*{{" + getText().toString().replaceAll("\\{", "\\\\{").replaceAll("\\[", "\\\\[").replaceAll("!", "\\\\!") + "}}*{color}";
+    }
+}

grammar/src/org/jetbrains/jet/grammar/SymbolToken.java

@@ -0,0 +1,15 @@
+package org.jetbrains.jet.grammar;
+
+/**
+ * @author abreslav
+ */
+public class SymbolToken extends Token {
+    public SymbolToken(CharSequence text) {
+        super(text);
+    }
+
+    @Override
+    public String toString() {
+        return "{color:blue}*" + getText().toString().replaceAll("\\*", "\\\\*") + "*{color}";
+    }
+}

grammar/src/org/jetbrains/jet/grammar/Token.java

@@ -0,0 +1,22 @@
+package org.jetbrains.jet.grammar;
+
+/**
+ * @author abreslav
+ */
+public class Token {
+    private final CharSequence text;
+
+    public Token(CharSequence text) {
+        this.text = text;
+    }
+
+    public CharSequence getText() {
+        return text;
+    }
+
+
+    @Override
+    public String toString() {
+        return getText().toString();
+    }
+}

grammar/src/org/jetbrains/jet/grammar/WhiteSpace.java

@@ -0,0 +1,15 @@
+package org.jetbrains.jet.grammar;
+
+/**
+ * @author abreslav
+ */
+public class WhiteSpace extends Token {
+    public WhiteSpace(CharSequence text) {
+        super(text);
+    }
+
+    @Override
+    public String toString() {
+        return super.toString().replaceAll(" ", " ");
+    }
+}

grammar/src/toplevel.grm

@@ -1,8 +1,15 @@
-[start] jetlFile
+/**
+h1. Syntax
+*/
+
+
+[start]
+jetFile
   : preamble toplevelObject*
   ;
 
-[start] script
+[start]
+script
   : preamble expression*
   ;
 

grammar/src/types.grm

@@ -1,3 +1,7 @@
+/**
+h2. Types
+*/
+
 /*
 
 Foo<Bar<X>, T, Object> // user type

grammar/src/when.grm

@@ -1,3 +1,7 @@
+/**
+h3. Pattern matching
+*/
+
 when
   : "when" "(" (modifiers "val" SimpleName "=")? expression ")" "{"
         whenEntry*
@@ -35,7 +39,7 @@ pattern
 decomposerPattern
   : type
 // TODO : typeParameters will be consumed by the expression
-  : elvisExpression typeParameters? '@' tuplePattern
+  : elvisExpression typeParameters? "@" tuplePattern
   ;
 
 constantPattern