From 626ec6519c05c5087f73c2be2fbca91eec80fc84 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 19 Mar 2014 11:40:38 -0400 Subject: [PATCH] Give tokens a value when possible When a token can only have one value, store that value along with the token. This is useful for unparsing kotlin. --- .../jetbrains/jet/lexer/JetKeywordToken.java | 21 ++-- .../jet/lexer/JetSingleValueToken.java | 36 +++++++ .../org/jetbrains/jet/lexer/JetTokens.java | 96 +++++++++---------- 3 files changed, 92 insertions(+), 61 deletions(-) create mode 100644 compiler/frontend/src/org/jetbrains/jet/lexer/JetSingleValueToken.java diff --git a/compiler/frontend/src/org/jetbrains/jet/lexer/JetKeywordToken.java b/compiler/frontend/src/org/jetbrains/jet/lexer/JetKeywordToken.java index c7771a11e60..6f54557d9b7 100644 --- a/compiler/frontend/src/org/jetbrains/jet/lexer/JetKeywordToken.java +++ b/compiler/frontend/src/org/jetbrains/jet/lexer/JetKeywordToken.java @@ -19,36 +19,33 @@ package org.jetbrains.jet.lexer; import org.jetbrains.annotations.NonNls; import org.jetbrains.annotations.NotNull; -public class JetKeywordToken extends JetToken { +public class JetKeywordToken extends JetSingleValueToken { /** * Generate keyword (identifier that has a keyword meaning in all possible contexts) */ public static JetKeywordToken keyword(String value) { - return new JetKeywordToken(value, false); + return keyword(value, value); + } + + public static JetKeywordToken keyword(String debugName, String value) { + return new JetKeywordToken(debugName, value, false); } /** * Generate soft keyword (identifier that has a keyword meaning only in some contexts) */ public static JetKeywordToken softKeyword(String value) { - return new JetKeywordToken(value, true); + return new JetKeywordToken(value, value, true); } - private final String myValue; private final boolean myIsSoft; - private JetKeywordToken(@NotNull @NonNls String value, boolean isSoft) { - super(value); - myValue = value; + private JetKeywordToken(@NotNull @NonNls String debugName, @NotNull @NonNls String value, boolean isSoft) { + super(debugName, value); myIsSoft = isSoft; } - public String getValue() { - return myValue; - } - - public boolean isSoft() { return myIsSoft; } diff --git a/compiler/frontend/src/org/jetbrains/jet/lexer/JetSingleValueToken.java b/compiler/frontend/src/org/jetbrains/jet/lexer/JetSingleValueToken.java new file mode 100644 index 00000000000..4597dd91058 --- /dev/null +++ b/compiler/frontend/src/org/jetbrains/jet/lexer/JetSingleValueToken.java @@ -0,0 +1,36 @@ +/* + * Copyright 2010-2014 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.jet.lexer; + +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; + +public class JetSingleValueToken extends JetToken { + + private final String myValue; + + public JetSingleValueToken(@NotNull @NonNls String debugName, @NotNull @NonNls String value) { + super(debugName); + myValue = value; + } + + @NotNull @NonNls + public String getValue() { + return myValue; + } + +} diff --git a/compiler/frontend/src/org/jetbrains/jet/lexer/JetTokens.java b/compiler/frontend/src/org/jetbrains/jet/lexer/JetTokens.java index 873f1ecbce2..561c6e81ed3 100644 --- a/compiler/frontend/src/org/jetbrains/jet/lexer/JetTokens.java +++ b/compiler/frontend/src/org/jetbrains/jet/lexer/JetTokens.java @@ -83,58 +83,56 @@ public interface JetTokens { JetToken LABEL_IDENTIFIER = new JetToken("LABEL_IDENTIFIER"); JetToken FIELD_IDENTIFIER = new JetToken("FIELD_IDENTIFIER"); - JetToken LBRACKET = new JetToken("LBRACKET"); - JetToken RBRACKET = new JetToken("RBRACKET"); - JetToken LBRACE = new JetToken("LBRACE"); - JetToken RBRACE = new JetToken("RBRACE"); - JetToken LPAR = new JetToken("LPAR"); - JetToken RPAR = new JetToken("RPAR"); - JetToken DOT = new JetToken("DOT"); - JetToken PLUSPLUS = new JetToken("PLUSPLUS"); - JetToken MINUSMINUS = new JetToken("MINUSMINUS"); - JetToken MUL = new JetToken("MUL"); - JetToken PLUS = new JetToken("PLUS"); - JetToken MINUS = new JetToken("MINUS"); - JetToken EXCL = new JetToken("EXCL"); - JetToken DIV = new JetToken("DIV"); - JetToken PERC = new JetToken("PERC"); - JetToken LT = new JetToken("LT"); - JetToken GT = new JetToken("GT"); - JetToken LTEQ = new JetToken("LTEQ"); - JetToken GTEQ = new JetToken("GTEQ"); - JetToken EQEQEQ = new JetToken("EQEQEQ"); - JetToken ARROW = new JetToken("ARROW"); - JetToken DOUBLE_ARROW = new JetToken("DOUBLE_ARROW"); - JetToken EXCLEQEQEQ = new JetToken("EXCLEQEQEQ"); - JetToken EQEQ = new JetToken("EQEQ"); - JetToken EXCLEQ = new JetToken("EXCLEQ"); - JetToken EXCLEXCL = new JetToken("EXCLEXCL"); - JetToken ANDAND = new JetToken("ANDAND"); - JetToken OROR = new JetToken("OROR"); - JetToken SAFE_ACCESS = new JetToken("SAFE_ACCESS"); - JetToken ELVIS = new JetToken("ELVIS"); - // JetToken MAP = new JetToken("MAP"); - // JetToken FILTER = new JetToken("FILTER"); - JetToken QUEST = new JetToken("QUEST"); - JetToken COLONCOLON = new JetToken("COLONCOLON"); - JetToken COLON = new JetToken("COLON"); - JetToken SEMICOLON = new JetToken("SEMICOLON"); - JetToken RANGE = new JetToken("RANGE"); - JetToken EQ = new JetToken("EQ"); - JetToken MULTEQ = new JetToken("MULTEQ"); - JetToken DIVEQ = new JetToken("DIVEQ"); - JetToken PERCEQ = new JetToken("PERCEQ"); - JetToken PLUSEQ = new JetToken("PLUSEQ"); - JetToken MINUSEQ = new JetToken("MINUSEQ"); - JetToken NOT_IN = JetKeywordToken.keyword("NOT_IN"); - JetToken NOT_IS = JetKeywordToken.keyword("NOT_IS"); - JetToken HASH = new JetToken("HASH"); - JetToken AT = new JetToken("AT"); - JetToken ATAT = new JetToken("ATAT"); + JetSingleValueToken LBRACKET = new JetSingleValueToken("LBRACKET", "["); + JetSingleValueToken RBRACKET = new JetSingleValueToken("RBRACKET", "]"); + JetSingleValueToken LBRACE = new JetSingleValueToken("LBRACE", "{"); + JetSingleValueToken RBRACE = new JetSingleValueToken("RBRACE", "}"); + JetSingleValueToken LPAR = new JetSingleValueToken("LPAR", "("); + JetSingleValueToken RPAR = new JetSingleValueToken("RPAR", ")"); + JetSingleValueToken DOT = new JetSingleValueToken("DOT", "."); + JetSingleValueToken PLUSPLUS = new JetSingleValueToken("PLUSPLUS", "++"); + JetSingleValueToken MINUSMINUS = new JetSingleValueToken("MINUSMINUS", "--"); + JetSingleValueToken MUL = new JetSingleValueToken("MUL", "*"); + JetSingleValueToken PLUS = new JetSingleValueToken("PLUS", "+"); + JetSingleValueToken MINUS = new JetSingleValueToken("MINUS", "-"); + JetSingleValueToken EXCL = new JetSingleValueToken("EXCL", "!"); + JetSingleValueToken DIV = new JetSingleValueToken("DIV", "/"); + JetSingleValueToken PERC = new JetSingleValueToken("PERC", "%"); + JetSingleValueToken LT = new JetSingleValueToken("LT", "<"); + JetSingleValueToken GT = new JetSingleValueToken("GT", ">"); + JetSingleValueToken LTEQ = new JetSingleValueToken("LTEQ", "<="); + JetSingleValueToken GTEQ = new JetSingleValueToken("GTEQ", ">="); + JetSingleValueToken EQEQEQ = new JetSingleValueToken("EQEQEQ", "=="); + JetSingleValueToken ARROW = new JetSingleValueToken("ARROW", "->"); + JetSingleValueToken DOUBLE_ARROW = new JetSingleValueToken("DOUBLE_ARROW", "=>"); + JetSingleValueToken EXCLEQEQEQ = new JetSingleValueToken("EXCLEQEQEQ", "!=="); + JetSingleValueToken EQEQ = new JetSingleValueToken("EQEQ", "=="); + JetSingleValueToken EXCLEQ = new JetSingleValueToken("EXCLEQ", "!="); + JetSingleValueToken EXCLEXCL = new JetSingleValueToken("EXCLEXCL", "!!"); + JetSingleValueToken ANDAND = new JetSingleValueToken("ANDAND", "&&"); + JetSingleValueToken OROR = new JetSingleValueToken("OROR", "||"); + JetSingleValueToken SAFE_ACCESS = new JetSingleValueToken("SAFE_ACCESS", "?."); + JetSingleValueToken ELVIS = new JetSingleValueToken("ELVIS", "?:"); + JetSingleValueToken QUEST = new JetSingleValueToken("QUEST", "?"); + JetSingleValueToken COLONCOLON = new JetSingleValueToken("COLONCOLON", "::"); + JetSingleValueToken COLON = new JetSingleValueToken("COLON", ":"); + JetSingleValueToken SEMICOLON = new JetSingleValueToken("SEMICOLON", ";"); + JetSingleValueToken RANGE = new JetSingleValueToken("RANGE", ".."); + JetSingleValueToken EQ = new JetSingleValueToken("EQ", "="); + JetSingleValueToken MULTEQ = new JetSingleValueToken("MULTEQ", "*="); + JetSingleValueToken DIVEQ = new JetSingleValueToken("DIVEQ", "/="); + JetSingleValueToken PERCEQ = new JetSingleValueToken("PERCEQ", "%="); + JetSingleValueToken PLUSEQ = new JetSingleValueToken("PLUSEQ", "+="); + JetSingleValueToken MINUSEQ = new JetSingleValueToken("MINUSEQ", "-="); + JetKeywordToken NOT_IN = JetKeywordToken.keyword("NOT_IN", "!in"); + JetKeywordToken NOT_IS = JetKeywordToken.keyword("NOT_IS", "!is"); + JetSingleValueToken HASH = new JetSingleValueToken("HASH", "#"); + JetSingleValueToken AT = new JetSingleValueToken("AT", "@"); + JetSingleValueToken ATAT = new JetSingleValueToken("ATAT", "@@"); TokenSet LABELS = TokenSet.create(AT, ATAT, LABEL_IDENTIFIER); - JetToken COMMA = new JetToken("COMMA"); + JetSingleValueToken COMMA = new JetSingleValueToken("COMMA", ","); JetToken EOL_OR_SEMICOLON = new JetToken("EOL_OR_SEMICOLON"); JetKeywordToken IMPORT_KEYWORD = JetKeywordToken.softKeyword("import");