azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Give tokens a value when possible

Browse Source 
When a token can only have one value, store that value along with the
token. This is useful for unparsing kotlin.
This commit is contained in:
Steven Allen
2014-03-19 11:40:38 -04:00
parent fa67ab481b
commit 626ec6519c
3 changed files with 92 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/frontend/src/org/jetbrains/jet/lexer/JetKeywordToken.java
+9 -12
View File
@@ -19,36 +19,33 @@ package org.jetbrains.jet.lexer;
import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
public class JetKeywordToken extends JetToken { public class JetKeywordToken extends JetSingleValueToken {
/** /**
* Generate keyword (identifier that has a keyword meaning in all possible contexts) * Generate keyword (identifier that has a keyword meaning in all possible contexts)
*/ */
public static JetKeywordToken keyword(String value) { public static JetKeywordToken keyword(String value) {
return new JetKeywordToken(value, false); return keyword(value, value);
}
public static JetKeywordToken keyword(String debugName, String value) {
return new JetKeywordToken(debugName, value, false);
} }
/** /**
* Generate soft keyword (identifier that has a keyword meaning only in some contexts) * Generate soft keyword (identifier that has a keyword meaning only in some contexts)
*/ */
public static JetKeywordToken softKeyword(String value) { public static JetKeywordToken softKeyword(String value) {
return new JetKeywordToken(value, true); return new JetKeywordToken(value, value, true);
} }
private final String myValue;
private final boolean myIsSoft; private final boolean myIsSoft;
private JetKeywordToken(@NotNull @NonNls String value, boolean isSoft) { private JetKeywordToken(@NotNull @NonNls String debugName, @NotNull @NonNls String value, boolean isSoft) {
super(value); super(debugName, value);
myValue = value;
myIsSoft = isSoft; myIsSoft = isSoft;
} }
public String getValue() {
return myValue;
}
public boolean isSoft() { public boolean isSoft() {
return myIsSoft; return myIsSoft;
} }
compiler/frontend/src/org/jetbrains/jet/lexer/JetSingleValueToken.java
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright 2010-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.jet.lexer;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
public class JetSingleValueToken extends JetToken {
private final String myValue;
public JetSingleValueToken(@NotNull @NonNls String debugName, @NotNull @NonNls String value) {
super(debugName);
myValue = value;
}
@NotNull @NonNls
public String getValue() {
return myValue;
}
}
compiler/frontend/src/org/jetbrains/jet/lexer/JetTokens.java
+47 -49
View File
@@ -83,58 +83,56 @@ public interface JetTokens {
JetToken LABEL_IDENTIFIER = new JetToken("LABEL_IDENTIFIER"); JetToken LABEL_IDENTIFIER = new JetToken("LABEL_IDENTIFIER");
JetToken FIELD_IDENTIFIER = new JetToken("FIELD_IDENTIFIER"); JetToken FIELD_IDENTIFIER = new JetToken("FIELD_IDENTIFIER");
JetToken LBRACKET = new JetToken("LBRACKET"); JetSingleValueToken LBRACKET = new JetSingleValueToken("LBRACKET", "[");
JetToken RBRACKET = new JetToken("RBRACKET"); JetSingleValueToken RBRACKET = new JetSingleValueToken("RBRACKET", "]");
JetToken LBRACE = new JetToken("LBRACE"); JetSingleValueToken LBRACE = new JetSingleValueToken("LBRACE", "{");
JetToken RBRACE = new JetToken("RBRACE"); JetSingleValueToken RBRACE = new JetSingleValueToken("RBRACE", "}");
JetToken LPAR = new JetToken("LPAR"); JetSingleValueToken LPAR = new JetSingleValueToken("LPAR", "(");
JetToken RPAR = new JetToken("RPAR"); JetSingleValueToken RPAR = new JetSingleValueToken("RPAR", ")");
JetToken DOT = new JetToken("DOT"); JetSingleValueToken DOT = new JetSingleValueToken("DOT", ".");
JetToken PLUSPLUS = new JetToken("PLUSPLUS"); JetSingleValueToken PLUSPLUS = new JetSingleValueToken("PLUSPLUS", "++");
JetToken MINUSMINUS = new JetToken("MINUSMINUS"); JetSingleValueToken MINUSMINUS = new JetSingleValueToken("MINUSMINUS", "--");
JetToken MUL = new JetToken("MUL"); JetSingleValueToken MUL = new JetSingleValueToken("MUL", "*");
JetToken PLUS = new JetToken("PLUS"); JetSingleValueToken PLUS = new JetSingleValueToken("PLUS", "+");
JetToken MINUS = new JetToken("MINUS"); JetSingleValueToken MINUS = new JetSingleValueToken("MINUS", "-");
JetToken EXCL = new JetToken("EXCL"); JetSingleValueToken EXCL = new JetSingleValueToken("EXCL", "!");
JetToken DIV = new JetToken("DIV"); JetSingleValueToken DIV = new JetSingleValueToken("DIV", "/");
JetToken PERC = new JetToken("PERC"); JetSingleValueToken PERC = new JetSingleValueToken("PERC", "%");
JetToken LT = new JetToken("LT"); JetSingleValueToken LT = new JetSingleValueToken("LT", "<");
JetToken GT = new JetToken("GT"); JetSingleValueToken GT = new JetSingleValueToken("GT", ">");
JetToken LTEQ = new JetToken("LTEQ"); JetSingleValueToken LTEQ = new JetSingleValueToken("LTEQ", "<=");
JetToken GTEQ = new JetToken("GTEQ"); JetSingleValueToken GTEQ = new JetSingleValueToken("GTEQ", ">=");
JetToken EQEQEQ = new JetToken("EQEQEQ"); JetSingleValueToken EQEQEQ = new JetSingleValueToken("EQEQEQ", "==");
JetToken ARROW = new JetToken("ARROW"); JetSingleValueToken ARROW = new JetSingleValueToken("ARROW", "->");
JetToken DOUBLE_ARROW = new JetToken("DOUBLE_ARROW"); JetSingleValueToken DOUBLE_ARROW = new JetSingleValueToken("DOUBLE_ARROW", "=>");
JetToken EXCLEQEQEQ = new JetToken("EXCLEQEQEQ"); JetSingleValueToken EXCLEQEQEQ = new JetSingleValueToken("EXCLEQEQEQ", "!==");
JetToken EQEQ = new JetToken("EQEQ"); JetSingleValueToken EQEQ = new JetSingleValueToken("EQEQ", "==");
JetToken EXCLEQ = new JetToken("EXCLEQ"); JetSingleValueToken EXCLEQ = new JetSingleValueToken("EXCLEQ", "!=");
JetToken EXCLEXCL = new JetToken("EXCLEXCL"); JetSingleValueToken EXCLEXCL = new JetSingleValueToken("EXCLEXCL", "!!");
JetToken ANDAND = new JetToken("ANDAND"); JetSingleValueToken ANDAND = new JetSingleValueToken("ANDAND", "&&");
JetToken OROR = new JetToken("OROR"); JetSingleValueToken OROR = new JetSingleValueToken("OROR", "||");
JetToken SAFE_ACCESS = new JetToken("SAFE_ACCESS"); JetSingleValueToken SAFE_ACCESS = new JetSingleValueToken("SAFE_ACCESS", "?.");
JetToken ELVIS = new JetToken("ELVIS"); JetSingleValueToken ELVIS = new JetSingleValueToken("ELVIS", "?:");
// JetToken MAP = new JetToken("MAP"); JetSingleValueToken QUEST = new JetSingleValueToken("QUEST", "?");
// JetToken FILTER = new JetToken("FILTER"); JetSingleValueToken COLONCOLON = new JetSingleValueToken("COLONCOLON", "::");
JetToken QUEST = new JetToken("QUEST"); JetSingleValueToken COLON = new JetSingleValueToken("COLON", ":");
JetToken COLONCOLON = new JetToken("COLONCOLON"); JetSingleValueToken SEMICOLON = new JetSingleValueToken("SEMICOLON", ";");
JetToken COLON = new JetToken("COLON"); JetSingleValueToken RANGE = new JetSingleValueToken("RANGE", "..");
JetToken SEMICOLON = new JetToken("SEMICOLON"); JetSingleValueToken EQ = new JetSingleValueToken("EQ", "=");
JetToken RANGE = new JetToken("RANGE"); JetSingleValueToken MULTEQ = new JetSingleValueToken("MULTEQ", "*=");
JetToken EQ = new JetToken("EQ"); JetSingleValueToken DIVEQ = new JetSingleValueToken("DIVEQ", "/=");
JetToken MULTEQ = new JetToken("MULTEQ"); JetSingleValueToken PERCEQ = new JetSingleValueToken("PERCEQ", "%=");
JetToken DIVEQ = new JetToken("DIVEQ"); JetSingleValueToken PLUSEQ = new JetSingleValueToken("PLUSEQ", "+=");
JetToken PERCEQ = new JetToken("PERCEQ"); JetSingleValueToken MINUSEQ = new JetSingleValueToken("MINUSEQ", "-=");
JetToken PLUSEQ = new JetToken("PLUSEQ"); JetKeywordToken NOT_IN = JetKeywordToken.keyword("NOT_IN", "!in");
JetToken MINUSEQ = new JetToken("MINUSEQ"); JetKeywordToken NOT_IS = JetKeywordToken.keyword("NOT_IS", "!is");
JetToken NOT_IN = JetKeywordToken.keyword("NOT_IN"); JetSingleValueToken HASH = new JetSingleValueToken("HASH", "#");
JetToken NOT_IS = JetKeywordToken.keyword("NOT_IS"); JetSingleValueToken AT = new JetSingleValueToken("AT", "@");
JetToken HASH = new JetToken("HASH"); JetSingleValueToken ATAT = new JetSingleValueToken("ATAT", "@@");
JetToken AT = new JetToken("AT");
JetToken ATAT = new JetToken("ATAT");
TokenSet LABELS = TokenSet.create(AT, ATAT, LABEL_IDENTIFIER); TokenSet LABELS = TokenSet.create(AT, ATAT, LABEL_IDENTIFIER);
JetToken COMMA = new JetToken("COMMA"); JetSingleValueToken COMMA = new JetSingleValueToken("COMMA", ",");
JetToken EOL_OR_SEMICOLON = new JetToken("EOL_OR_SEMICOLON"); JetToken EOL_OR_SEMICOLON = new JetToken("EOL_OR_SEMICOLON");
JetKeywordToken IMPORT_KEYWORD = JetKeywordToken.softKeyword("import"); JetKeywordToken IMPORT_KEYWORD = JetKeywordToken.softKeyword("import");