parser, baby steps. Soft keywords highlighting

This commit is contained in:
Maxim Shafirov
2010-12-13 19:04:03 +03:00
parent 4f18686978
commit 17d9240d14
11 changed files with 533 additions and 30 deletions
+2
View File
@@ -3,7 +3,9 @@
<component name="CodeStyleSettingsManager">
<option name="PER_PROJECT_SETTINGS">
<value>
<option name="ELSE_ON_NEW_LINE" value="true" />
<option name="ALIGN_GROUP_FIELD_DECLARATIONS" value="true" />
<option name="IF_BRACE_FORCE" value="1" />
<ADDITIONAL_INDENT_OPTIONS fileType="groovy">
<option name="INDENT_SIZE" value="2" />
<option name="CONTINUATION_INDENT_SIZE" value="8" />
+1
View File
@@ -20,5 +20,6 @@
<fileTypeFactory implementation="org.jetbrains.jet.lang.JetFileFactory"/>
<lang.braceMatcher language="jet" implementationClass="org.jetbrains.jet.lang.JetPairMatcher"/>
<lang.parserDefinition language="jet" implementationClass="org.jetbrains.jet.lang.parsing.JetParserDefinition"/>
<annotator language="jet" implementationClass="org.jetbrains.jet.lang.annotations.SoftKeywordsAnnotator"/>
</extensions>
</idea-plugin>
@@ -0,0 +1,17 @@
/*
* @author max
*/
package org.jetbrains.jet;
import com.intellij.lang.Language;
import com.intellij.psi.tree.IElementType;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jet.lang.JetLanguage;
public class JetNodeType extends IElementType {
public JetNodeType(@NotNull @NonNls String debugName) {
super(debugName, JetLanguage.INSTANCE);
}
}
+21 -1
View File
@@ -3,10 +3,30 @@
*/
package org.jetbrains.jet;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.IFileElementType;
import org.jetbrains.jet.lang.JetLanguage;
public interface JetNodeTypes {
IFileElementType JET_FILE_NODE = new IFileElementType(JetLanguage.INSTANCE);
IFileElementType JET_FILE = new IFileElementType(JetLanguage.INSTANCE);
JetNodeType NAMESPACE = new JetNodeType("NAMESPACE");
JetNodeType CLASS = new JetNodeType("CLASS");
JetNodeType PROPERTY = new JetNodeType("PROPERTY");
JetNodeType FUN = new JetNodeType("FUN");
JetNodeType EXTENSION = new JetNodeType("EXTENSION");
JetNodeType TYPEDEF = new JetNodeType("TYPEDEF");
JetNodeType TYPE_PARAMETER_LIST = new JetNodeType("TYPE_PARAMETER_LIST");
JetNodeType TYPE_PARAMETER = new JetNodeType("TYPE_PARAMETER");
JetNodeType PRIMARY_CONSTRUCTOR_PARAMETERS_LIST = new JetNodeType("PRIMARY_CONSTRUCTOR_PARAMETERS_LIST");
JetNodeType PRIMARY_CONSTRUCTOR_PARAMETER = new JetNodeType("PRIMARY_CONSTRUCTOR_PARAMETER");
JetNodeType DELEGATION_SPECIFIER_LIST = new JetNodeType("DELEGATION_SPECIFIER_LIST");
JetNodeType DELEGATION_SPECIFIER = new JetNodeType("DELEGATION_SPECIFIER");
JetNodeType DELEGATOR_BY = new JetNodeType("DELEGATOR_BY");
JetNodeType DELEGATOR_SUPER_CALL = new JetNodeType("DELEGATOR_SUPER_CALL");
JetNodeType DELEGATOR_SUPER_CLASS = new JetNodeType("DELEGATOR_SUPER_CLASS");
JetNodeType VALUE_PARAMETER_LIST = new JetNodeType("VALUE_PARAMETER_LIST");
JetNodeType NAMED_ARGUMENT = new JetNodeType("NAMED_ARGUMENT");
JetNodeType CLASS_BODY = new JetNodeType("CLASS_BODY");
}
@@ -25,7 +25,7 @@ public class JetHighlighter extends SyntaxHighlighterBase {
SyntaxHighlighterColors.KEYWORD.getDefaultAttributes()
);
static final TextAttributesKey JET_SOFT_KEYWORD = TextAttributesKey.createTextAttributesKey(
public static final TextAttributesKey JET_SOFT_KEYWORD = TextAttributesKey.createTextAttributesKey(
"JET.SOFT.KEYWORD",
SyntaxHighlighterColors.KEYWORD.getDefaultAttributes()
);
@@ -0,0 +1,22 @@
/*
* @author max
*/
package org.jetbrains.jet.lang.annotations;
import com.intellij.lang.annotation.AnnotationHolder;
import com.intellij.lang.annotation.Annotator;
import com.intellij.psi.PsiElement;
import com.intellij.psi.impl.source.tree.LeafPsiElement;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jet.lang.JetHighlighter;
import org.jetbrains.jet.lexer.JetTokens;
public class SoftKeywordsAnnotator implements Annotator {
public void annotate(@NotNull PsiElement element, @NotNull AnnotationHolder holder) {
if (element instanceof LeafPsiElement) {
if (JetTokens.SOFT_KEYWORDS.contains(((LeafPsiElement) element).getElementType())) {
holder.createInfoAnnotation(element, null).setTextAttributes(JetHighlighter.JET_SOFT_KEYWORD);
}
}
}
}
@@ -13,12 +13,7 @@ import org.jetbrains.jet.JetNodeTypes;
public class JetParser implements PsiParser {
@NotNull
public ASTNode parse(IElementType iElementType, PsiBuilder psiBuilder) {
PsiBuilder.Marker mark = psiBuilder.mark();
while (!psiBuilder.eof()) {
psiBuilder.advanceLexer();
}
mark.done(JetNodeTypes.JET_FILE_NODE);
new JetParsing(psiBuilder).parseFile();
return psiBuilder.getTreeBuilt();
}
}
@@ -12,14 +12,12 @@ import com.intellij.openapi.project.Project;
import com.intellij.psi.FileViewProvider;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiFile;
import com.intellij.psi.TokenType;
import com.intellij.psi.tree.IFileElementType;
import com.intellij.psi.tree.TokenSet;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.jet.JetNodeTypes;
import org.jetbrains.jet.lang.psi.JetFile;
import org.jetbrains.jet.lexer.JetLexer;
import org.jetbrains.jet.lexer.JetToken;
import org.jetbrains.jet.lexer.JetTokens;
public class JetParserDefinition implements ParserDefinition {
@@ -33,7 +31,7 @@ public class JetParserDefinition implements ParserDefinition {
}
public IFileElementType getFileNodeType() {
return JetNodeTypes.JET_FILE_NODE;
return JetNodeTypes.JET_FILE;
}
@NotNull
@@ -54,7 +52,7 @@ public class JetParserDefinition implements ParserDefinition {
@NotNull
public PsiElement createElement(ASTNode astNode) {
// TODO
return new ASTWrapperPsiElement(astNode);
return new CompositeNode(astNode);
}
public PsiFile createFile(FileViewProvider fileViewProvider) {
@@ -64,4 +62,15 @@ public class JetParserDefinition implements ParserDefinition {
public SpaceRequirements spaceExistanceTypeBetweenTokens(ASTNode astNode, ASTNode astNode1) {
return SpaceRequirements.MAY;
}
private static class CompositeNode extends ASTWrapperPsiElement {
private CompositeNode(@NotNull ASTNode node) {
super(node);
}
@Override
public String toString() {
return getNode().getElementType().toString();
}
}
}
@@ -0,0 +1,417 @@
/*
* @author max
*/
package org.jetbrains.jet.lang.parsing;
import com.intellij.lang.ITokenTypeRemapper;
import com.intellij.lang.PsiBuilder;
import com.intellij.psi.tree.IElementType;
import com.intellij.psi.tree.TokenSet;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.jet.JetNodeType;
import org.jetbrains.jet.lexer.JetSoftKeywordToken;
import org.jetbrains.jet.lexer.JetToken;
import org.jetbrains.jet.lexer.JetTokens;
import static org.jetbrains.jet.JetNodeTypes.*;
import static org.jetbrains.jet.lexer.JetTokens.*;
public class JetParsing {
public static final TokenSet CLASS_NAME_FOLLOW = TokenSet.create(LT, WRAPS_KEYWORD, LPAR, COLON, LBRACE);
public static final TokenSet TYPE_PARAMETER_GT_FOLLOW = TokenSet.create(WHERE_KEYWORD, WRAPS_KEYWORD, LPAR, COLON, LBRACE, GT);
public static final TokenSet PARAMETER_NAME_FOLLOW = TokenSet.create(COLON, EQ, COMMA, RPAR);
private final PsiBuilder myBuilder;
public JetParsing(PsiBuilder myBuilder) {
this.myBuilder = myBuilder;
}
public void parseFile() {
PsiBuilder.Marker fileMarker = mark();
PsiBuilder.Marker namespaceMarker = parsePreamble();
while (!eof()) {
parseTopLevelObject();
}
if (namespaceMarker != null) {
namespaceMarker.done(NAMESPACE);
}
fileMarker.done(JET_FILE);
}
@Nullable
private PsiBuilder.Marker parsePreamble() {
if (at(NAMESPACE_KEYWORD)) {
advance();
PsiBuilder.Marker namespaceMarker = mark();
parseQualifiedName();
return namespaceMarker;
}
while (at(IMPORT_KEYWORD)) {
parseImportDirective();
}
return null;
}
private void parseImportDirective() {
// TODO
}
private PsiBuilder.Marker mark() {
return myBuilder.mark();
}
private void parseQualifiedName() {
while (true) {
if (tt() != IDENTIFIER) {
if (tt() != DOT) {
error("Expecting qualified name");
break;
}
error("Qualified name must start with an identifier");
advance();
}
else {
advance();
if (tt() != DOT) break;
advance();
}
}
}
private void error(String message) {
myBuilder.error(message);
}
private void parseTopLevelObject() {
PsiBuilder.Marker decl = mark();
if (!parseModifierList()) {
decl.drop();
return;
}
JetToken keywordToken = tt();
JetNodeType declType = null;
if (keywordToken == CLASS_KEYWORD) {
declType = parseClass();
}
else if (keywordToken == EXTENSION_KEYWORD) {
declType = parseExtension();
}
else if (keywordToken == FUN_KEYWORD) {
declType = parseFunction();
}
else if (keywordToken == VAL_KEYWORD || keywordToken == VAR_KEYWORD) {
declType = parseProperty();
}
else if (keywordToken == NAMESPACE_KEYWORD) {
declType = parseBlockNamespace();
}
else if (keywordToken == TYPE_KEYWORD) {
declType = parseTypedef();
}
if (declType == null) {
errorAndAdvance("Expecting namespace or top level declaration");
decl.drop();
}
else {
decl.done(declType);
}
}
private JetNodeType parseTypedef() {
return TYPEDEF;
}
private JetNodeType parseBlockNamespace() {
return NAMESPACE;
}
private JetNodeType parseProperty() {
return PROPERTY;
}
private JetNodeType parseFunction() {
return FUN;
}
private JetNodeType parseExtension() {
return EXTENSION;
}
private JetNodeType parseClass() {
assert at(CLASS_KEYWORD);
advance();
expect(IDENTIFIER, "Class name expected", CLASS_NAME_FOLLOW);
parseTypeParameterList();
if (at(WRAPS_KEYWORD)) advance();
if (at(LPAR)) {
parsePrimaryConstructorParameterList();
}
if (at(COLON)) {
advance();
parseDelegationSpecifierList();
}
if (at(LBRACE)) {
parseClassBody();
}
return CLASS;
}
private void parseClassBody() {
assert at(LBRACE);
PsiBuilder.Marker body = mark();
advance();
while (!eof()) {
if (at(RBRACE)) {
break;
}
advance();
}
expect(RBRACE, "Missing '}");
body.done(CLASS_BODY);
}
private void parseDelegationSpecifierList() {
PsiBuilder.Marker list = mark();
while (true) {
parseDelegationSpecifier();
if (!at(COMMA)) break;
advance();
}
list.done(DELEGATION_SPECIFIER_LIST);
}
private void parseDelegationSpecifier() {
PsiBuilder.Marker specifier = mark();
parseAttributeList();
PsiBuilder.Marker delegator = mark();
parseTypeRef(); // TODO: Error recovery!!!
if (at(BY_KEYWORD)) {
advance();
parseExpression();
delegator.done(DELEGATOR_BY);
}
else if (at(LPAR)) {
parseValueParameterList();
delegator.done(DELEGATOR_SUPER_CALL);
}
else {
delegator.done(DELEGATOR_SUPER_CLASS);
}
specifier.done(DELEGATION_SPECIFIER);
}
private void parseValueParameterList() {
assert at(LPAR);
PsiBuilder.Marker list = mark();
advance();
while (true) {
if (at(IDENTIFIER) && lookahead(1) == EQ) {
PsiBuilder.Marker named = mark();
advance();
advance();
parseExpression();
named.done(NAMED_ARGUMENT);
}
else {
parseExpression();
}
if (!at(COMMA)) {
break;
}
}
expect(RPAR, "Missing ')'");
list.done(VALUE_PARAMETER_LIST);
}
private void parseAttributeList() {
// TODO
}
private void parsePrimaryConstructorParameterList() {
assert at(LPAR);
PsiBuilder.Marker cons = mark();
advance();
while (true) {
parsePrimaryConstructorParameter();
if (!at(COMMA)) break;
advance();
}
expect(RPAR, "')' expected");
cons.done(PRIMARY_CONSTRUCTOR_PARAMETERS_LIST);
}
private void parsePrimaryConstructorParameter() {
PsiBuilder.Marker param = mark();
parseModifierList();
if (at(VAR_KEYWORD) || at(VAL_KEYWORD)) {
advance();
}
parseFunctionParameterRest();
param.done(PRIMARY_CONSTRUCTOR_PARAMETER);
}
private void parseFunctionParameterRest() {
expect(IDENTIFIER, "Parameter name expected", PARAMETER_NAME_FOLLOW);
if (at(COLON)) {
advance();
parseTypeRef();
}
else {
error("Parameters must have type annotation");
}
if (at(EQ)) {
advance();
parseExpression();
}
}
private void parseExpression() {
advance();
// TODO
}
private void parseTypeParameterList() {
PsiBuilder.Marker list = mark();
if (tt() == LT) {
advance();
while (true) {
parseTypeParameter();
if (!at(COMMA)) break;
advance();
}
expect(GT, "Missing '>'", TYPE_PARAMETER_GT_FOLLOW);
}
list.done(TYPE_PARAMETER_LIST);
}
private void parseTypeParameter() {
if (TYPE_PARAMETER_GT_FOLLOW.contains(tt())) {
error("Type parameter declaration expected");
return;
}
PsiBuilder.Marker mark = mark();
parseModifierList();
expect(IDENTIFIER, "Type parameter name expected", TokenSet.EMPTY);
if (at(COLON)) {
advance();
parseTypeRef();
}
mark.done(TYPE_PARAMETER);
}
private void parseTypeRef() {
advance();
// tODO:
}
private boolean expect(JetToken expectation, String message) {
return expect(expectation, message, null);
}
private boolean expect(JetToken expectation, String message, TokenSet recoverySet) {
JetToken tt = tt();
if (tt == expectation) {
advance();
return true;
}
if (recoverySet == null || recoverySet.contains(tt)) {
error(message);
}
else {
errorAndAdvance(message);
}
return false;
}
private void errorAndAdvance(String message) {
PsiBuilder.Marker err = mark();
advance();
err.error(message);
}
private boolean parseModifierList() {
// TODO
return true;
}
private boolean eof() {
return myBuilder.eof();
}
private void advance() {
myBuilder.advanceLexer();
}
private JetToken tt() {
return (JetToken) myBuilder.getTokenType();
}
private boolean at(final IElementType expectation) {
JetToken token = tt();
if (token == expectation) return true;
if (token == IDENTIFIER && expectation instanceof JetSoftKeywordToken) {
if (((JetSoftKeywordToken) expectation).getValue().equals(myBuilder.getTokenText())) {
myBuilder.setTokenTypeRemapper(new ITokenTypeRemapper() {
public IElementType filter(IElementType source, int start, int end, CharSequence text) {
return expectation;
}
});
tt();
myBuilder.setTokenTypeRemapper(null);
return true;
}
}
return false;
}
private IElementType lookahead(int k) {
PsiBuilder.Marker tmp = mark();
for (int i = 0; i < k; i++) advance();
JetToken tt = tt();
tmp.rollbackTo();
return tt;
}
}
@@ -0,0 +1,20 @@
/*
* @author max
*/
package org.jetbrains.jet.lexer;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
public class JetSoftKeywordToken extends JetToken {
private final String myValue;
public JetSoftKeywordToken(@NotNull @NonNls String value) {
super(value);
myValue = value;
}
public String getValue() {
return myValue;
}
}
+18 -18
View File
@@ -98,24 +98,24 @@ public interface JetTokens {
JetToken MINUSEQ = new JetToken("MINUSEQ");
JetToken COMMA = new JetToken("COMMA");
JetToken WRAPS_KEYWORD = new JetToken("wraps");
JetToken WHERE_KEYWORD = new JetToken("where");
JetToken BY_KEYWORD = new JetToken("by");
JetToken LAZY_KEYWORD = new JetToken("lazy");
JetToken GET_KEYWORD = new JetToken("get");
JetToken SET_KEYWORD = new JetToken("set");
JetToken ABSTRACT_KEYWORD = new JetToken("abstract");
JetToken VIRTUAL_KEYWORD = new JetToken("virtual");
JetToken ENUM_KEYWORD = new JetToken("enum");
JetToken OPEN_KEYWORD = new JetToken("open");
JetToken ATTRIBUTE_KEYWORD = new JetToken("attribute");
JetToken OVERRIDE_KEYWORD = new JetToken("override");
JetToken PRIVATE_KEYWORD = new JetToken("private");
JetToken PUBLIC_KEYWORD = new JetToken("public");
JetToken INTERNAL_KEYWORD = new JetToken("internal");
JetToken PROTECTED_KEYWORD = new JetToken("protected");
JetToken OUT_KEYWORD = new JetToken("out");
JetToken REF_KEYWORD = new JetToken("ref");
JetSoftKeywordToken WRAPS_KEYWORD = new JetSoftKeywordToken("wraps");
JetSoftKeywordToken WHERE_KEYWORD = new JetSoftKeywordToken("where");
JetSoftKeywordToken BY_KEYWORD = new JetSoftKeywordToken("by");
JetSoftKeywordToken LAZY_KEYWORD = new JetSoftKeywordToken("lazy");
JetSoftKeywordToken GET_KEYWORD = new JetSoftKeywordToken("get");
JetSoftKeywordToken SET_KEYWORD = new JetSoftKeywordToken("set");
JetSoftKeywordToken ABSTRACT_KEYWORD = new JetSoftKeywordToken("abstract");
JetSoftKeywordToken VIRTUAL_KEYWORD = new JetSoftKeywordToken("virtual");
JetSoftKeywordToken ENUM_KEYWORD = new JetSoftKeywordToken("enum");
JetSoftKeywordToken OPEN_KEYWORD = new JetSoftKeywordToken("open");
JetSoftKeywordToken ATTRIBUTE_KEYWORD = new JetSoftKeywordToken("attribute");
JetSoftKeywordToken OVERRIDE_KEYWORD = new JetSoftKeywordToken("override");
JetSoftKeywordToken PRIVATE_KEYWORD = new JetSoftKeywordToken("private");
JetSoftKeywordToken PUBLIC_KEYWORD = new JetSoftKeywordToken("public");
JetSoftKeywordToken INTERNAL_KEYWORD = new JetSoftKeywordToken("internal");
JetSoftKeywordToken PROTECTED_KEYWORD = new JetSoftKeywordToken("protected");
JetSoftKeywordToken OUT_KEYWORD = new JetSoftKeywordToken("out");
JetSoftKeywordToken REF_KEYWORD = new JetSoftKeywordToken("ref");
TokenSet KEYWORDS = TokenSet.create(NAMESPACE_KEYWORD, IMPORT_KEYWORD, AS_KEYWORD, TYPE_KEYWORD, CLASS_KEYWORD,
THIS_KEYWORD, VAL_KEYWORD, VAR_KEYWORD, FUN_KEYWORD, DECOMPOSER_KEYWORD, EXTENSION_KEYWORD, FOR_KEYWORD,