File names + proper lines

This commit is contained in:
Andrey Breslav
2012-08-31 16:11:05 +04:00
parent 366b069679
commit 15ce394000
13 changed files with 152 additions and 135 deletions
@@ -20,8 +20,8 @@ package org.jetbrains.jet.grammar;
* @author abreslav
*/
public class Annotation extends Token {
public Annotation(CharSequence text, int line) {
super(text, line);
public Annotation(CharSequence text, String fileName, int line) {
super(text, fileName, line);
}
@Override
@@ -20,8 +20,8 @@ package org.jetbrains.jet.grammar;
* @author abreslav
*/
public class Comment extends Token {
public Comment(CharSequence text, int line) {
super(text, line);
public Comment(CharSequence text, String fileName, int line) {
super(text, fileName, line);
}
@Override
@@ -17,10 +17,8 @@
package org.jetbrains.jet.grammar;
import com.google.common.base.Supplier;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Sets;
import com.google.common.collect.*;
import com.intellij.openapi.util.io.FileUtil;
import java.awt.*;
import java.awt.datatransfer.Clipboard;
@@ -28,7 +26,6 @@ import java.awt.datatransfer.ClipboardOwner;
import java.awt.datatransfer.StringSelection;
import java.awt.datatransfer.Transferable;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.*;
@@ -61,35 +58,32 @@ public class ConfluenceHyperlinksGenerator {
File grammarDir = new File("grammar/src");
Set<File> used = new HashSet<File>();
StringBuilder allRules = getAllRulesInOneString(grammarDir, used);
List<Token> tokens = getJoinedTokensFromAllFiles(grammarDir, used);
assertAllFilesAreUsed(grammarDir, used);
StringBuilder output = markDeclarations(allRules);
StringBuilder result = generate(output);
StringBuilder result = generate(tokens);
copyToClipboard(result);
}
private static StringBuilder getAllRulesInOneString(File grammarDir, Set<File> used) throws IOException {
StringBuilder allRules = new StringBuilder();
private static List<Token> getJoinedTokensFromAllFiles(File grammarDir, Set<File> used) throws IOException {
List<Token> allTokens = Lists.newArrayList();
for (String fileName : FILE_NAMES_IN_ORDER) {
File file = new File(grammarDir, fileName + "." + GRAMMAR_EXTENSION);
used.add(file);
FileReader fileReader = new FileReader(file);
try {
int c;
while ((c = fileReader.read()) != -1) {
allRules.append((char) c);
}
allRules.append("\n");
allRules.append("\n");
} finally {
fileReader.close();
}
String text = FileUtil.loadFile(file, true);
StringBuilder textWithMarkedDeclarations = markDeclarations(text);
List<Token> tokens = tokenize(createLexer(file.getPath(), textWithMarkedDeclarations));
allTokens.addAll(tokens);
}
return allRules;
return allTokens;
}
private static _GrammarLexer createLexer(String fileName, StringBuilder output) {
_GrammarLexer grammarLexer = new _GrammarLexer((Reader) null);
grammarLexer.reset(output, 0, output.length(), 0);
grammarLexer.setFileName(fileName);
return grammarLexer;
}
private static void assertAllFilesAreUsed(File grammarDir, Set<File> used) {
@@ -102,7 +96,7 @@ public class ConfluenceHyperlinksGenerator {
}
}
private static StringBuilder markDeclarations(StringBuilder allRules) {
private static StringBuilder markDeclarations(CharSequence allRules) {
StringBuilder output = new StringBuilder();
Pattern symbolReference = Pattern.compile("^\\w+$", Pattern.MULTILINE);
@@ -120,11 +114,7 @@ public class ConfluenceHyperlinksGenerator {
return output;
}
private static StringBuilder generate(StringBuilder output) throws IOException {
_GrammarLexer grammarLexer = new _GrammarLexer((Reader) null);
grammarLexer
.reset(output, 0, output.length(), 0);
private static StringBuilder generate(List<Token> tokens) throws IOException {
StringBuilder result = new StringBuilder("h1. Contents\n").append("{toc:style=disc|indent=20px}");
Set<String> declaredSymbols = new HashSet<String>();
@@ -137,7 +127,6 @@ public class ConfluenceHyperlinksGenerator {
}
});
List<Token> tokens = tokenize(grammarLexer);
Declaration lastDeclaration = null;
for (Token advance: tokens) {
if (advance instanceof Declaration) {
@@ -159,7 +148,7 @@ public class ConfluenceHyperlinksGenerator {
result.append("{anchor:").append(declaration.getName()).append("}");
if (!usedSymbols.contains(declaration.getName())) {
// result.append("(!) *Unused!* ");
System.out.println("Unused: " + token + " at line " + token.getLine());
System.out.println("Unused: " + tokenWithPosition(token));
}
Collection<String> myUsages = usages.get(declaration.getName());
if (!myUsages.isEmpty()) {
@@ -180,7 +169,7 @@ public class ConfluenceHyperlinksGenerator {
Identifier identifier = (Identifier) token;
if (!declaredSymbols.contains(identifier.getName())) {
result.append("(!) *Undeclared!* ");
System.out.println("Undeclared: " + token + " at line " + token.getLine());
System.out.println("Undeclared: " + tokenWithPosition(token));
}
}
result.append(token);
@@ -188,6 +177,10 @@ public class ConfluenceHyperlinksGenerator {
return result;
}
private static String tokenWithPosition(Token token) {
return token + " at " + token.getFileName() + ":" + token.getLine();
}
private static List<Token> tokenize(_GrammarLexer grammarLexer) throws IOException {
List<Token> tokens = new ArrayList<Token>();
while (true) {
@@ -22,8 +22,8 @@ package org.jetbrains.jet.grammar;
public class Declaration extends Token {
private final String name;
public Declaration(CharSequence text, int line) {
super(text, line);
public Declaration(CharSequence text, String fileName, int line) {
super(text, fileName, line);
name = text.toString().substring(1);
}
@@ -20,8 +20,8 @@ package org.jetbrains.jet.grammar;
* @author abreslav
*/
public class DocComment extends Token {
public DocComment(CharSequence text, int line) {
super(text, line);
public DocComment(CharSequence text, String fileName, int line) {
super(text, fileName, line);
}
@Override
@@ -12,10 +12,22 @@ package org.jetbrains.jet.grammar;
%unicode
%class _GrammarLexer
%{
private String fileName;
public void setFileName(String fileName) {
this.fileName = fileName;
}
private int line = 1;
public int getCurrentLine() {
return line;
}
private void computeLine() {
CharSequence s = yytext();
for (int i = 0; i < s.length(); i++) {
if (s.charAt(i) == '\n') line++;
}
}
%}
%function advance
%type Token
@@ -77,33 +89,32 @@ RAW_STRING_LITERAL = {THREE_QUO} {QUO_STRING_CHAR}* {THREE_QUO}?
%%
<YYINITIAL> {BLOCK_COMMENT} { return new Comment(yytext(), line); }
<YYINITIAL> {DOC_COMMENT} { return new DocComment(yytext(), line); }
<YYINITIAL> {BLOCK_COMMENT} { computeLine(); return new Comment(yytext(), fileName, line); }
<YYINITIAL> {DOC_COMMENT} { computeLine(); return new DocComment(yytext(), fileName, line); }
<YYINITIAL> "\n" | "\r" { line++; return new WhiteSpace(yytext(), line); }
<YYINITIAL> ({WHITE_SPACE_CHAR})+ { return new WhiteSpace(yytext(), line); }
<YYINITIAL> ({WHITE_SPACE_CHAR})+ { computeLine(); return new WhiteSpace(yytext(), fileName, line); }
<YYINITIAL> {EOL_COMMENT} { return new Comment(yytext(), line); }
<YYINITIAL> {EOL_COMMENT} { computeLine(); return new Comment(yytext(), fileName, line); }
<YYINITIAL> {STRING_LITERAL} { return new StringToken(yytext(), line); }
<YYINITIAL> {ANGLE_STRING_LITERAL} { return new StringToken(yytext(), line); }
<YYINITIAL> {IDENTIFIER} { return new Identifier(yytext(), line); }
<YYINITIAL> "[" {IDENTIFIER} "]" { return new Annotation(yytext(), line); }
<YYINITIAL> {DECLARATION_IDENTIFIER} { return new Declaration(yytext(), line); }
<YYINITIAL> {STRING_LITERAL} { return new StringToken(yytext(), fileName, line); }
<YYINITIAL> {ANGLE_STRING_LITERAL} { return new StringToken(yytext(), fileName, line); }
<YYINITIAL> {IDENTIFIER} { return new Identifier(yytext(), fileName, line); }
<YYINITIAL> "[" {IDENTIFIER} "]" { return new Annotation(yytext(), fileName, line); }
<YYINITIAL> {DECLARATION_IDENTIFIER} { return new Declaration(yytext(), fileName, line); }
<YYINITIAL> ":" { return new SymbolToken(yytext(), line); }
<YYINITIAL> "{" { return new SymbolToken("\\" + yytext(), line); }
<YYINITIAL> "}" { return new SymbolToken("\\" + yytext(), line); }
<YYINITIAL> "[" { return new SymbolToken("\\" + yytext(), line); }
<YYINITIAL> "]" { return new SymbolToken("\\" + yytext(), line); }
<YYINITIAL> "(" { return new SymbolToken("\\" + yytext(), line); }
<YYINITIAL> ")" { return new SymbolToken("\\" + yytext(), line); }
<YYINITIAL> "*" { return new SymbolToken(yytext(), line); }
<YYINITIAL> "+" { return new SymbolToken(yytext(), line); }
<YYINITIAL> "?" { return new SymbolToken(yytext(), line); }
<YYINITIAL> "|" { return new SymbolToken(yytext(), line); }
<YYINITIAL> "-" { return new SymbolToken(yytext(), line); }
<YYINITIAL> "." { return new SymbolToken(yytext(), line); }
<YYINITIAL> ":" { return new SymbolToken(yytext(), fileName, line); }
<YYINITIAL> "{" { return new SymbolToken("\\" + yytext(), fileName, line); }
<YYINITIAL> "}" { return new SymbolToken("\\" + yytext(), fileName, line); }
<YYINITIAL> "[" { return new SymbolToken("\\" + yytext(), fileName, line); }
<YYINITIAL> "]" { return new SymbolToken("\\" + yytext(), fileName, line); }
<YYINITIAL> "(" { return new SymbolToken("\\" + yytext(), fileName, line); }
<YYINITIAL> ")" { return new SymbolToken("\\" + yytext(), fileName, line); }
<YYINITIAL> "*" { return new SymbolToken(yytext(), fileName, line); }
<YYINITIAL> "+" { return new SymbolToken(yytext(), fileName, line); }
<YYINITIAL> "?" { return new SymbolToken(yytext(), fileName, line); }
<YYINITIAL> "|" { return new SymbolToken(yytext(), fileName, line); }
<YYINITIAL> "-" { return new SymbolToken(yytext(), fileName, line); }
<YYINITIAL> "." { return new SymbolToken(yytext(), fileName, line); }
<YYINITIAL> . { return new Other(yytext(), line); }
<YYINITIAL> . { return new Other(yytext(), fileName, line); }
@@ -22,8 +22,8 @@ package org.jetbrains.jet.grammar;
public class Identifier extends Token {
private final String name;
public Identifier(CharSequence text, int line) {
super(text, line);
public Identifier(CharSequence text, String fileName, int line) {
super(text, fileName, line);
name = text.toString();
}
@@ -20,7 +20,7 @@ package org.jetbrains.jet.grammar;
* @author abreslav
*/
public class Other extends Token {
public Other(CharSequence text, int line) {
super(text, line);
public Other(CharSequence text, String fileName, int line) {
super(text, fileName, line);
}
}
@@ -20,8 +20,8 @@ package org.jetbrains.jet.grammar;
* @author abreslav
*/
public class StringToken extends Token {
public StringToken(CharSequence text, int line) {
super(text, line);
public StringToken(CharSequence text, String fileName, int line) {
super(text, fileName, line);
}
@Override
@@ -20,8 +20,8 @@ package org.jetbrains.jet.grammar;
* @author abreslav
*/
public class SymbolToken extends Token {
public SymbolToken(CharSequence text, int line) {
super(text, line);
public SymbolToken(CharSequence text, String fileName, int line) {
super(text, fileName, line);
}
@Override
@@ -20,11 +20,13 @@ package org.jetbrains.jet.grammar;
* @author abreslav
*/
public class Token {
private final String fileName;
private final CharSequence text;
private final int line;
public Token(CharSequence text, int line) {
public Token(CharSequence text, String fileName, int line) {
this.text = text;
this.fileName = fileName;
this.line = line;
}
@@ -32,6 +34,10 @@ public class Token {
return text;
}
public String getFileName() {
return fileName;
}
public int getLine() {
return line;
}
@@ -20,8 +20,8 @@ package org.jetbrains.jet.grammar;
* @author abreslav
*/
public class WhiteSpace extends Token {
public WhiteSpace(CharSequence text, int line) {
super(text, line);
public WhiteSpace(CharSequence text, String fileName, int line) {
super(text, fileName, line);
}
@Override
@@ -1,4 +1,4 @@
/* The following code was generated by JFlex 1.4.3 on 8/31/12 2:58 PM */
/* The following code was generated by JFlex 1.4.3 on 8/31/12 4:09 PM */
/* It's an automatically generated code. Do not modify it. */
package org.jetbrains.jet.grammar;
@@ -13,7 +13,7 @@ package org.jetbrains.jet.grammar;
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.3
* on 8/31/12 2:58 PM from the specification file
* on 8/31/12 4:09 PM from the specification file
* <tt>/Users/abreslav/work/kotlin/grammar/src/org/jetbrains/jet/grammar/Grammar.flex</tt>
*/
class _GrammarLexer {
@@ -37,13 +37,13 @@ class _GrammarLexer {
* Translates characters to character classes
*/
private static final String ZZ_CMAP_PACKED =
"\11\1\1\3\1\10\1\0\1\3\1\16\16\1\4\0\1\3\1\0"+
"\1\13\1\0\1\2\1\0\1\5\1\0\1\21\1\21\1\7\1\11"+
"\11\1\1\3\1\10\1\0\1\3\1\0\16\1\4\0\1\3\1\0"+
"\1\13\1\0\1\2\1\0\1\5\1\0\1\20\1\20\1\7\1\11"+
"\1\0\1\11\1\11\1\6\1\1\11\1\1\11\1\0\1\14\1\0"+
"\1\15\1\11\1\0\1\2\1\2\2\2\1\2\1\2\5\2\1\2"+
"\3\2\1\2\7\2\1\2\2\2\1\17\1\12\1\20\1\0\1\2"+
"\3\2\1\2\7\2\1\2\2\2\1\16\1\12\1\17\1\0\1\2"+
"\1\4\1\2\1\2\2\2\1\2\1\2\5\2\1\2\3\2\1\2"+
"\7\2\1\2\2\2\1\21\1\11\1\21\1\0\41\1\2\0\4\2"+
"\7\2\1\2\2\2\1\20\1\11\1\20\1\0\41\1\2\0\4\2"+
"\4\0\1\2\2\0\1\1\7\0\1\2\4\0\1\2\5\0\27\2"+
"\1\0\37\2\1\0\u013f\2\31\0\162\2\4\0\14\2\16\0\5\2"+
"\11\0\1\2\21\0\130\1\5\0\23\1\12\0\1\2\13\0\1\2"+
@@ -137,12 +137,12 @@ class _GrammarLexer {
private static final int [] ZZ_ACTION = zzUnpackAction();
private static final String ZZ_ACTION_PACKED_0 =
"\1\0\1\1\1\2\1\3\3\1\1\4\1\5\2\6"+
"\1\5\2\7\1\0\1\10\2\11\2\6\2\0\1\2"+
"\1\11\1\12\1\13\2\0\2\12\1\0\1\11\1\0";
"\1\0\1\1\1\2\1\3\3\1\1\4\2\5\2\6"+
"\1\0\1\7\2\10\2\5\2\0\1\2\1\10\1\11"+
"\1\12\2\0\2\11\1\0\1\10\1\0";
private static int [] zzUnpackAction() {
int [] result = new int[33];
int [] result = new int[31];
int offset = 0;
offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
return result;
@@ -167,14 +167,13 @@ class _GrammarLexer {
private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
private static final String ZZ_ROWMAP_PACKED_0 =
"\0\0\0\22\0\44\0\66\0\110\0\132\0\154\0\22"+
"\0\66\0\176\0\220\0\22\0\242\0\22\0\264\0\306"+
"\0\330\0\352\0\374\0\22\0\u010e\0\u0120\0\22\0\u0132"+
"\0\u0144\0\22\0\u0156\0\u0168\0\u017a\0\22\0\u018c\0\22"+
"\0\u019e";
"\0\0\0\21\0\42\0\63\0\104\0\125\0\146\0\21"+
"\0\167\0\210\0\231\0\21\0\252\0\273\0\314\0\335"+
"\0\356\0\21\0\377\0\u0110\0\21\0\u0121\0\u0132\0\21"+
"\0\u0143\0\u0154\0\u0165\0\21\0\u0176\0\21\0\u0187";
private static int [] zzUnpackRowMap() {
int [] result = new int[33];
int [] result = new int[31];
int offset = 0;
offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
return result;
@@ -197,21 +196,21 @@ class _GrammarLexer {
private static final int [] ZZ_TRANS = zzUnpackTrans();
private static final String ZZ_TRANS_PACKED_0 =
"\2\2\1\3\1\4\1\5\1\6\1\7\1\10\1\11"+
"\1\10\1\2\1\12\1\13\1\2\1\14\1\15\2\16"+
"\23\0\2\3\22\0\1\4\4\0\1\4\13\0\1\17"+
"\21\0\1\20\25\0\1\21\1\22\12\0\10\12\1\0"+
"\1\12\1\23\1\24\6\12\10\13\1\0\1\13\1\24"+
"\2\13\1\24\4\13\2\0\1\25\1\0\1\26\16\0"+
"\2\17\1\0\1\27\16\0\2\20\17\0\10\21\1\0"+
"\11\21\7\30\1\31\12\30\10\12\1\0\11\12\1\0"+
"\2\25\15\0\1\32\3\0\1\33\17\0\7\30\1\34"+
"\12\30\6\35\1\36\1\31\12\35\1\0\2\33\1\0"+
"\1\37\15\0\6\30\1\40\1\34\12\30\7\35\1\41"+
"\12\35\20\0\1\32\1\0\6\35\1\36\1\41\12\35";
"\2\2\1\3\1\4\1\5\1\6\1\7\1\10\1\4"+
"\1\10\1\2\1\11\1\12\1\2\1\13\2\14\22\0"+
"\2\3\21\0\1\4\4\0\1\4\12\0\1\15\20\0"+
"\1\16\24\0\1\17\1\20\11\0\10\11\1\0\1\11"+
"\1\21\1\22\5\11\10\12\1\0\1\12\1\22\2\12"+
"\1\22\3\12\2\0\1\23\1\0\1\24\15\0\2\15"+
"\1\0\1\25\15\0\2\16\16\0\10\17\1\0\10\17"+
"\7\26\1\27\11\26\10\11\1\0\10\11\1\0\2\23"+
"\14\0\1\30\3\0\1\31\16\0\7\26\1\32\11\26"+
"\6\33\1\34\1\27\11\33\1\0\2\31\1\0\1\35"+
"\14\0\6\26\1\36\1\32\11\26\7\33\1\37\11\33"+
"\17\0\1\30\1\0\6\33\1\34\1\37\11\33";
private static int [] zzUnpackTrans() {
int [] result = new int[432];
int [] result = new int[408];
int offset = 0;
offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
return result;
@@ -252,12 +251,12 @@ class _GrammarLexer {
private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
private static final String ZZ_ATTRIBUTE_PACKED_0 =
"\1\0\1\11\5\1\1\11\3\1\1\11\1\1\1\11"+
"\1\0\4\1\1\11\2\0\1\11\2\1\1\11\2\0"+
"\1\1\1\11\1\0\1\11\1\0";
"\1\0\1\11\5\1\1\11\3\1\1\11\1\0\4\1"+
"\1\11\2\0\1\11\2\1\1\11\2\0\1\1\1\11"+
"\1\0\1\11\1\0";
private static int [] zzUnpackAttribute() {
int [] result = new int[33];
int [] result = new int[31];
int offset = 0;
offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
return result;
@@ -316,11 +315,23 @@ class _GrammarLexer {
private boolean zzEOFDone;
/* user code: */
private String fileName;
public void setFileName(String fileName) {
this.fileName = fileName;
}
private int line = 1;
public int getCurrentLine() {
return line;
}
private void computeLine() {
CharSequence s = yytext();
for (int i = 0; i < s.length(); i++) {
if (s.charAt(i) == '\n') line++;
}
}
_GrammarLexer(java.io.Reader in) {
this.zzReader = in;
@@ -568,50 +579,46 @@ class _GrammarLexer {
zzMarkedPos = zzMarkedPosL;
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
case 10:
{ return new DocComment(yytext(), line);
case 7:
{ return new Declaration(yytext(), fileName, line);
}
case 11: break;
case 4:
{ return new SymbolToken(yytext(), fileName, line);
}
case 12: break;
case 11:
{ return new Annotation(yytext(), line);
case 6:
{ return new SymbolToken("\\" + yytext(), fileName, line);
}
case 13: break;
case 5:
{ line++; return new WhiteSpace(yytext(), line);
case 3:
{ computeLine(); return new WhiteSpace(yytext(), fileName, line);
}
case 14: break;
case 2:
{ return new Identifier(yytext(), line);
case 9:
{ computeLine(); return new DocComment(yytext(), fileName, line);
}
case 15: break;
case 3:
{ return new WhiteSpace(yytext(), line);
case 1:
{ return new Other(yytext(), fileName, line);
}
case 16: break;
case 1:
{ return new Other(yytext(), line);
case 10:
{ return new Annotation(yytext(), fileName, line);
}
case 17: break;
case 4:
{ return new SymbolToken(yytext(), line);
case 8:
{ computeLine(); return new Comment(yytext(), fileName, line);
}
case 18: break;
case 6:
{ return new StringToken(yytext(), line);
case 2:
{ return new Identifier(yytext(), fileName, line);
}
case 19: break;
case 8:
{ return new Declaration(yytext(), line);
case 5:
{ return new StringToken(yytext(), fileName, line);
}
case 20: break;
case 7:
{ return new SymbolToken("\\" + yytext(), line);
}
case 21: break;
case 9:
{ return new Comment(yytext(), line);
}
case 22: break;
default:
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
zzAtEOF = true;