Parse violation source code.
This commit is contained in:
Kui LIU
@@ -1,104 +0,0 @@
|
||||
package edu.lu.uni.serval.bugLocalization;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
|
||||
import edu.lu.uni.serval.FixPatternParser.Tokenizer;
|
||||
import edu.lu.uni.serval.FixPatternParser.violations.Violation;
|
||||
import edu.lu.uni.serval.config.Configuration;
|
||||
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
|
||||
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
|
||||
import edu.lu.uni.serval.utils.FileHelper;
|
||||
|
||||
public class ParseAlarms {
|
||||
|
||||
public static void main(String[] args) {
|
||||
/*
|
||||
* Alarm java files.
|
||||
* Position files.
|
||||
*/
|
||||
String fixedAlarmFilesPath = Configuration.GUM_TREE_INPUT + "prevFiles/";
|
||||
String positionsFilePath = Configuration.GUM_TREE_INPUT + "positions/";
|
||||
|
||||
String unfixedAlarmFilesPath = Configuration.GUM_TREE_INPUT + "unfixAlarms/";
|
||||
String un_positionsFilePath = Configuration.GUM_TREE_INPUT + "un_positions/";
|
||||
|
||||
StringBuilder tokensBuilder = new StringBuilder();
|
||||
List<File> javaFiles = FileHelper.getAllFilesInCurrentDiectory(fixedAlarmFilesPath, ".java");
|
||||
int counter = 0;
|
||||
for (File javaFile : javaFiles) {
|
||||
String fileName = javaFile.getName().replace(".java", ".txt");
|
||||
fileName = fileName.substring(5);
|
||||
|
||||
List<Violation> violations = readViolations(positionsFilePath + fileName);
|
||||
|
||||
|
||||
for (Violation violation : violations) {
|
||||
int startLine = violation.getStartLineNum();
|
||||
int endLine = violation.getEndLineNum();
|
||||
String alarmType = violation.getAlarmType();
|
||||
|
||||
if (endLine > startLine + 5) continue;
|
||||
|
||||
AlarmTree alarmTree = new AlarmTree(javaFile, startLine, endLine);
|
||||
alarmTree.extract();
|
||||
List<ITree> matchedTrees = alarmTree.getAlarmTrees();
|
||||
SimpleTree simpleTree = new SimpleTree();
|
||||
simpleTree.setLabel("Block");
|
||||
simpleTree.setNodeType("Block");
|
||||
List<SimpleTree> children = new ArrayList<>();
|
||||
|
||||
for (ITree matchedTree : matchedTrees) {
|
||||
SimpleTree simpleT = new SimplifyTree().canonicalizeSourceCodeTree(matchedTree, null);
|
||||
children.add(simpleT);
|
||||
}
|
||||
simpleTree.setChildren(children);
|
||||
|
||||
String tokens = Tokenizer.getTokensDeepFirst(simpleTree);
|
||||
tokensBuilder.append(fileName + "\n" + alarmType + ": " + tokens + "\n");
|
||||
counter ++;
|
||||
if (counter % 100 == 0) {
|
||||
FileHelper.outputToFile(Configuration.ROOT_PATH + "Alarms_tokens/fixedAlarms.list", tokensBuilder, true);
|
||||
tokensBuilder.setLength(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
FileHelper.outputToFile(Configuration.ROOT_PATH + "Alarms_tokens/fixedAlarms.list", tokensBuilder, true);
|
||||
tokensBuilder.setLength(0);
|
||||
}
|
||||
|
||||
private static List<Violation> readViolations(String file) {
|
||||
List<Violation> violations = new ArrayList<>();
|
||||
|
||||
String fileContent = FileHelper.readFile(file);
|
||||
BufferedReader reader = null;
|
||||
reader = new BufferedReader(new StringReader(fileContent));
|
||||
String line = null;
|
||||
try {
|
||||
while ((line = reader.readLine()) != null) {
|
||||
String[] positionStr = line.split(":");
|
||||
int startLine = Integer.parseInt(positionStr[0]);
|
||||
int endLine = Integer.parseInt(positionStr[1]);
|
||||
String alarmType = positionStr[2];
|
||||
Violation violation = new Violation(startLine, endLine, alarmType);
|
||||
violations.add(violation);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
try {
|
||||
reader.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return violations;
|
||||
}
|
||||
}
|
||||
+89
-16
@@ -1,4 +1,4 @@
|
||||
package edu.lu.uni.serval.bugLocalization;
|
||||
package edu.lu.uni.serval.violation.code.parser;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
@@ -49,7 +49,7 @@ public class AlarmTree {
|
||||
public int getAlarmFinalEndLine() {
|
||||
return alarmFinalEndLine;
|
||||
}
|
||||
|
||||
|
||||
public void extract() {
|
||||
ITree rootTree = new GumTreeGenerator().generateITreeForJavaFile(file, GumTreeType.EXP_JDT);
|
||||
|
||||
@@ -62,7 +62,7 @@ public class AlarmTree {
|
||||
}
|
||||
|
||||
int endPosition = startPosition + tree.getLength();
|
||||
int endLine = cUnit.getLineNumber(endPosition);
|
||||
int endLine = cUnit.getLineNumber(endPosition - 1);
|
||||
if (endLine < alarmStartLine) continue;
|
||||
|
||||
matchTrees(tree.getChildren());
|
||||
@@ -73,6 +73,38 @@ public class AlarmTree {
|
||||
this.alarmFinalStartLine = cUnit.getLineNumber(this.matchedTrees.get(0).getPos());
|
||||
ITree lastTree = matchedTrees.get(size - 1);
|
||||
this.alarmFinalEndLine = cUnit.getLineNumber(lastTree.getPos() + lastTree.getLength());
|
||||
} else {
|
||||
System.err.println(this.file.getName() + "===" + this.alarmStartLine + ":" + this.alarmEndLine);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void extract(String type) {
|
||||
ITree rootTree = new GumTreeGenerator().generateITreeForJavaFile(file, GumTreeType.EXP_JDT);
|
||||
|
||||
List<ITree> trees = rootTree.getChildren();
|
||||
for (ITree tree : trees) {
|
||||
int startPosition = tree.getPos();
|
||||
int startLine = cUnit.getLineNumber(startPosition);
|
||||
if (startLine > alarmEndLine) {
|
||||
break;
|
||||
}
|
||||
|
||||
int endPosition = startPosition + tree.getLength();
|
||||
int endLine = cUnit.getLineNumber(endPosition - 1);
|
||||
if (endLine < alarmStartLine) continue;
|
||||
|
||||
matchTrees(tree.getChildren());
|
||||
}
|
||||
|
||||
int size = matchedTrees.size();
|
||||
if (size > 0) {
|
||||
this.alarmFinalStartLine = cUnit.getLineNumber(this.matchedTrees.get(0).getPos());
|
||||
ITree lastTree = matchedTrees.get(size - 1);
|
||||
this.alarmFinalEndLine = cUnit.getLineNumber(lastTree.getPos() + lastTree.getLength());
|
||||
} else {
|
||||
System.err.println(type);
|
||||
System.err.println(this.file.getName() + "===" + this.alarmStartLine + ":" + this.alarmEndLine);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -89,32 +121,69 @@ public class AlarmTree {
|
||||
int endLine = cUnit.getLineNumber(endPosition);
|
||||
if (endLine < alarmStartLine) continue;
|
||||
|
||||
if (startLine >= alarmStartLine) {
|
||||
if (!isStatement(tree)) {
|
||||
if (endLine == alarmEndLine) {
|
||||
if (tree.getType() == 31) { // MethodDeclaration
|
||||
matchTrees(tree.getChildren());
|
||||
} else if (isStatement(tree)) {
|
||||
addToMatchedTrees(tree);
|
||||
} else {
|
||||
ITree parent = getParentStatement(tree);
|
||||
if (parent == null) {
|
||||
if (tree.getLabel().equals("MethodBody") || tree.getType() == 8) {
|
||||
if (tree.getType() == 8) { // 8: Block
|
||||
matchTrees(tree.getChildren());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
tree = parent;
|
||||
addToMatchedTrees(parent);
|
||||
}
|
||||
if (!matchedTrees.contains(tree)) {
|
||||
matchedTrees.add(tree);
|
||||
}
|
||||
if (containsBlockStatement(tree)) {
|
||||
endLine = cUnit.getLineNumber(tree.getPos() + tree.getLength());
|
||||
if (endLine > alarmEndLine) {
|
||||
tree = removeBlock(tree);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (startLine >= alarmStartLine) {
|
||||
if (isStatement(tree)) {
|
||||
addToMatchedTrees(tree);
|
||||
} else {
|
||||
ITree parent = getParentStatement(tree);
|
||||
if (parent == null) {
|
||||
if (tree.getType() == 8) {
|
||||
matchTrees(tree.getChildren());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
addToMatchedTrees(parent);
|
||||
}
|
||||
} else {
|
||||
// if (tree.getType() == 14) {
|
||||
// ITree parent = getParentStatement(tree);
|
||||
// if (parent == null) {
|
||||
// matchTrees(tree.getChildren());
|
||||
// } else {
|
||||
// addToMatchedTrees(parent);
|
||||
// }
|
||||
// } else {
|
||||
// matchTrees(tree.getChildren());
|
||||
// }
|
||||
matchTrees(tree.getChildren());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void addToMatchedTrees(ITree tree) {
|
||||
if (!matchedTrees.contains(tree)) {
|
||||
/*
|
||||
* TODO with the same parent, or the sub trees.
|
||||
* In the same method body.
|
||||
*/
|
||||
matchedTrees.add(tree);
|
||||
}
|
||||
if (containsBlockStatement(tree)) {
|
||||
int endLine = cUnit.getLineNumber(tree.getPos() + tree.getLength());
|
||||
if (endLine > alarmEndLine) {
|
||||
tree = removeBlock(tree);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private ITree removeBlock(ITree tree) {
|
||||
List<ITree> oldChildren = tree.getChildren();
|
||||
List<ITree> newChildren = new ArrayList<>();
|
||||
@@ -127,7 +196,7 @@ public class AlarmTree {
|
||||
int endPosition = startPosition + child.getLength();
|
||||
int endLine = cUnit.getLineNumber(endPosition);
|
||||
if (endLine > this.alarmEndLine) {
|
||||
if (child.getType() == 8 || containsBlockStatement(child)) {
|
||||
if (child.getType() == 8 || containsBlockStatement(child)) { // 8: Block
|
||||
child = removeBlock(child);
|
||||
if (child.getChildren().size() == 0) {
|
||||
continue;
|
||||
@@ -144,8 +213,12 @@ public class AlarmTree {
|
||||
ITree parent = tree;
|
||||
do {
|
||||
parent = parent.getParent();
|
||||
if (parent == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
int type = parent.getType();
|
||||
if (type == 1 || type == 31 || type == 55) {
|
||||
if (type == 1 || type == 31 || type == 55 || type == 71) {
|
||||
// AnonymousClassDeclaration
|
||||
// MethodDeclaration Initializer (type == 28)
|
||||
// TypeDeclaration
|
||||
@@ -0,0 +1,147 @@
|
||||
package edu.lu.uni.serval.violation.code.parser;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
|
||||
import edu.lu.uni.serval.FixPatternParser.Tokenizer;
|
||||
import edu.lu.uni.serval.FixPatternParser.violations.Violation;
|
||||
import edu.lu.uni.serval.config.Configuration;
|
||||
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
|
||||
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
|
||||
import edu.lu.uni.serval.utils.FileHelper;
|
||||
|
||||
public class ViolationCodeParser {
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(ViolationCodeParser.class);
|
||||
|
||||
public static void main(String[] args) {
|
||||
/*
|
||||
* Fixed violations:
|
||||
* Violation code files.
|
||||
* Position files.
|
||||
*/
|
||||
String fixedViolationFilesPath = Configuration.GUM_TREE_INPUT + "prevFiles/";
|
||||
String positionsFilePath = Configuration.GUM_TREE_INPUT + "positions/";
|
||||
int subIndex = 5;
|
||||
String outputPath = Configuration.ROOT_PATH + "Alarms_tokens/fixedAlarms.list";
|
||||
FileHelper.deleteFile(outputPath);
|
||||
new ViolationCodeParser().parse(fixedViolationFilesPath, positionsFilePath, subIndex, outputPath);
|
||||
|
||||
/*
|
||||
* UnFixed violations:
|
||||
* Violation code files.
|
||||
* Position files.
|
||||
*/
|
||||
String unfixedViolationFilesPath = Configuration.GUM_TREE_INPUT + "unfixAlarms/";
|
||||
String un_positionsFilePath = Configuration.GUM_TREE_INPUT + "un_positions/";
|
||||
subIndex = 8;
|
||||
outputPath = Configuration.ROOT_PATH + "Alarms_tokens/unfixedAlarms.list";
|
||||
FileHelper.deleteFile(outputPath);
|
||||
new ViolationCodeParser().parse(unfixedViolationFilesPath, un_positionsFilePath, subIndex, outputPath);
|
||||
}
|
||||
|
||||
public void parse(String alarmFilesPath, String positionFilesPath, int subIndex, String outputPath) {
|
||||
StringBuilder tokensBuilder = new StringBuilder();
|
||||
List<File> javaFiles = FileHelper.getAllFilesInCurrentDiectory(alarmFilesPath, ".java");
|
||||
int counter = 0;
|
||||
int a = 0;
|
||||
int maxLength = 0;
|
||||
for (File javaFile : javaFiles) {
|
||||
String fileName = javaFile.getName().replace(".java", ".txt");
|
||||
fileName = fileName.substring(subIndex);
|
||||
|
||||
List<Violation> violations = readViolationInfo(positionFilesPath + fileName);
|
||||
|
||||
for (Violation violation : violations) {
|
||||
int startLine = violation.getStartLineNum();
|
||||
int endLine = violation.getEndLineNum();
|
||||
String alarmType = violation.getAlarmType();
|
||||
|
||||
// if (endLine > startLine + 5) {
|
||||
// log.warn("#Large_Violation_Hunk: " + fileName.replace("#", "/").replace(".txt", ".java") + ":" + startLine + ":" + endLine + ":" + alarmType);
|
||||
// continue;
|
||||
// }
|
||||
|
||||
AlarmTree alarmTree = new AlarmTree(javaFile, startLine, endLine);
|
||||
alarmTree.extract();
|
||||
List<ITree> matchedTrees = alarmTree.getAlarmTrees();
|
||||
if (matchedTrees.size() == 0) {
|
||||
System.out.println(fileName + " == " + startLine + " : " + endLine);
|
||||
a ++;
|
||||
log.warn("#Null_Violation_Hunk: " + fileName.replace("#", "/").replace(".txt", ".java") + ":" + startLine + ":" + endLine + ":" + alarmType);
|
||||
continue;
|
||||
}
|
||||
SimpleTree simpleTree = new SimpleTree();
|
||||
simpleTree.setLabel("Block");
|
||||
simpleTree.setNodeType("Block");
|
||||
List<SimpleTree> children = new ArrayList<>();
|
||||
|
||||
for (ITree matchedTree : matchedTrees) {
|
||||
SimpleTree simpleT = new SimplifyTree().canonicalizeSourceCodeTree(matchedTree, null);
|
||||
children.add(simpleT);
|
||||
}
|
||||
simpleTree.setChildren(children);
|
||||
|
||||
String tokens = Tokenizer.getTokensDeepFirst(simpleTree);
|
||||
String[] tokensArray = tokens.split(" ");
|
||||
int length = tokensArray.length;
|
||||
if (length > maxLength) maxLength = length;
|
||||
tokensBuilder.append(alarmType + ":" + fileName + ":" + alarmTree.getAlarmFinalStartLine() + ":" + alarmTree.getAlarmFinalEndLine() + ":" + tokens + "\n");
|
||||
counter ++;
|
||||
if (counter % 5000 == 0) {
|
||||
FileHelper.outputToFile(outputPath, tokensBuilder, true);
|
||||
tokensBuilder.setLength(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
System.out.println(counter);
|
||||
System.out.println(a);
|
||||
System.out.println("MaxLength: " + maxLength);
|
||||
FileHelper.outputToFile(outputPath, tokensBuilder, true);
|
||||
tokensBuilder.setLength(0);
|
||||
}
|
||||
|
||||
private List<Violation> readViolationInfo(String file) {
|
||||
List<Violation> violations = new ArrayList<>();
|
||||
|
||||
String fileContent = FileHelper.readFile(file);
|
||||
BufferedReader reader = null;
|
||||
reader = new BufferedReader(new StringReader(fileContent));
|
||||
String line = null;
|
||||
try {
|
||||
while ((line = reader.readLine()) != null) {
|
||||
String[] positionStr = line.split(":");
|
||||
int startLine = Integer.parseInt(positionStr[1]);
|
||||
int endLine = Integer.parseInt(positionStr[2]);
|
||||
String alarmType = positionStr[0];
|
||||
|
||||
if (startLine == -1 || endLine == -1) {
|
||||
log.warn("#Illegal_Line_Position: " + FileHelper.getFileName(file).replace("#", "/").replace(".txt", ".java") + ":" + startLine + ":" + endLine + ":" + alarmType);
|
||||
continue;
|
||||
}
|
||||
|
||||
Violation violation = new Violation(startLine, endLine, alarmType);
|
||||
violations.add(violation);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
try {
|
||||
reader.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return violations;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user