azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parse violation source code.

Browse Source
This commit is contained in:
Kui LIU
2017-09-11 11:21:42 +02:00
parent c8351169f7
commit 6647961ac3
3 changed files with 236 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/bugLocalization/ParseAlarms.java
-104
View File
@@ -1,104 +0,0 @@
package edu.lu.uni.serval.bugLocalization;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.FixPatternParser.Tokenizer;
import edu.lu.uni.serval.FixPatternParser.violations.Violation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
import edu.lu.uni.serval.utils.FileHelper;
public class ParseAlarms {
public static void main(String[] args) {
/*
* Alarm java files.
* Position files.
*/
String fixedAlarmFilesPath = Configuration.GUM_TREE_INPUT + "prevFiles/";
String positionsFilePath = Configuration.GUM_TREE_INPUT + "positions/";
String unfixedAlarmFilesPath = Configuration.GUM_TREE_INPUT + "unfixAlarms/";
String un_positionsFilePath = Configuration.GUM_TREE_INPUT + "un_positions/";
StringBuilder tokensBuilder = new StringBuilder();
List<File> javaFiles = FileHelper.getAllFilesInCurrentDiectory(fixedAlarmFilesPath, ".java");
int counter = 0;
for (File javaFile : javaFiles) {
String fileName = javaFile.getName().replace(".java", ".txt");
fileName = fileName.substring(5);
List<Violation> violations = readViolations(positionsFilePath + fileName);
for (Violation violation : violations) {
int startLine = violation.getStartLineNum();
int endLine = violation.getEndLineNum();
String alarmType = violation.getAlarmType();
if (endLine > startLine + 5) continue;
AlarmTree alarmTree = new AlarmTree(javaFile, startLine, endLine);
alarmTree.extract();
List<ITree> matchedTrees = alarmTree.getAlarmTrees();
SimpleTree simpleTree = new SimpleTree();
simpleTree.setLabel("Block");
simpleTree.setNodeType("Block");
List<SimpleTree> children = new ArrayList<>();
for (ITree matchedTree : matchedTrees) {
SimpleTree simpleT = new SimplifyTree().canonicalizeSourceCodeTree(matchedTree, null);
children.add(simpleT);
}
simpleTree.setChildren(children);
String tokens = Tokenizer.getTokensDeepFirst(simpleTree);
tokensBuilder.append(fileName + "\n" + alarmType + ": " + tokens + "\n");
counter ++;
if (counter % 100 == 0) {
FileHelper.outputToFile(Configuration.ROOT_PATH + "Alarms_tokens/fixedAlarms.list", tokensBuilder, true);
tokensBuilder.setLength(0);
}
}
}
FileHelper.outputToFile(Configuration.ROOT_PATH + "Alarms_tokens/fixedAlarms.list", tokensBuilder, true);
tokensBuilder.setLength(0);
}
private static List<Violation> readViolations(String file) {
List<Violation> violations = new ArrayList<>();
String fileContent = FileHelper.readFile(file);
BufferedReader reader = null;
reader = new BufferedReader(new StringReader(fileContent));
String line = null;
try {
while ((line = reader.readLine()) != null) {
String[] positionStr = line.split(":");
int startLine = Integer.parseInt(positionStr[0]);
int endLine = Integer.parseInt(positionStr[1]);
String alarmType = positionStr[2];
Violation violation = new Violation(startLine, endLine, alarmType);
violations.add(violation);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return violations;
}
}
src/main/java/edu/lu/uni/serval/bugLocalization/AlarmTree.java → src/main/java/edu/lu/uni/serval/violation/code/parser/AlarmTree.java
+88 -15
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.bugLocalization; package edu.lu.uni.serval.violation.code.parser;
import java.io.File; import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
@@ -62,7 +62,7 @@ public class AlarmTree {
} }
int endPosition = startPosition + tree.getLength(); int endPosition = startPosition + tree.getLength();
int endLine = cUnit.getLineNumber(endPosition); int endLine = cUnit.getLineNumber(endPosition - 1);
if (endLine < alarmStartLine) continue; if (endLine < alarmStartLine) continue;
matchTrees(tree.getChildren()); matchTrees(tree.getChildren());
@@ -73,6 +73,38 @@ public class AlarmTree {
this.alarmFinalStartLine = cUnit.getLineNumber(this.matchedTrees.get(0).getPos()); this.alarmFinalStartLine = cUnit.getLineNumber(this.matchedTrees.get(0).getPos());
ITree lastTree = matchedTrees.get(size - 1); ITree lastTree = matchedTrees.get(size - 1);
this.alarmFinalEndLine = cUnit.getLineNumber(lastTree.getPos() + lastTree.getLength()); this.alarmFinalEndLine = cUnit.getLineNumber(lastTree.getPos() + lastTree.getLength());
} else {
System.err.println(this.file.getName() + "===" + this.alarmStartLine + ":" + this.alarmEndLine);
}
}
public void extract(String type) {
ITree rootTree = new GumTreeGenerator().generateITreeForJavaFile(file, GumTreeType.EXP_JDT);
List<ITree> trees = rootTree.getChildren();
for (ITree tree : trees) {
int startPosition = tree.getPos();
int startLine = cUnit.getLineNumber(startPosition);
if (startLine > alarmEndLine) {
break;
}
int endPosition = startPosition + tree.getLength();
int endLine = cUnit.getLineNumber(endPosition - 1);
if (endLine < alarmStartLine) continue;
matchTrees(tree.getChildren());
}
int size = matchedTrees.size();
if (size > 0) {
this.alarmFinalStartLine = cUnit.getLineNumber(this.matchedTrees.get(0).getPos());
ITree lastTree = matchedTrees.get(size - 1);
this.alarmFinalEndLine = cUnit.getLineNumber(lastTree.getPos() + lastTree.getLength());
} else {
System.err.println(type);
System.err.println(this.file.getName() + "===" + this.alarmStartLine + ":" + this.alarmEndLine);
} }
} }
@@ -89,32 +121,69 @@ public class AlarmTree {
int endLine = cUnit.getLineNumber(endPosition); int endLine = cUnit.getLineNumber(endPosition);
if (endLine < alarmStartLine) continue; if (endLine < alarmStartLine) continue;
if (startLine >= alarmStartLine) { if (endLine == alarmEndLine) {
if (!isStatement(tree)) { if (tree.getType() == 31) { // MethodDeclaration
matchTrees(tree.getChildren());
} else if (isStatement(tree)) {
addToMatchedTrees(tree);
} else {
ITree parent = getParentStatement(tree); ITree parent = getParentStatement(tree);
if (parent == null) { if (parent == null) {
if (tree.getLabel().equals("MethodBody") || tree.getType() == 8) { if (tree.getType() == 8) { // 8: Block
matchTrees(tree.getChildren()); matchTrees(tree.getChildren());
} }
continue; continue;
} }
tree = parent; addToMatchedTrees(parent);
} }
if (!matchedTrees.contains(tree)) { continue;
matchedTrees.add(tree); }
}
if (containsBlockStatement(tree)) { if (startLine >= alarmStartLine) {
endLine = cUnit.getLineNumber(tree.getPos() + tree.getLength()); if (isStatement(tree)) {
if (endLine > alarmEndLine) { addToMatchedTrees(tree);
tree = removeBlock(tree); } else {
ITree parent = getParentStatement(tree);
if (parent == null) {
if (tree.getType() == 8) {
matchTrees(tree.getChildren());
}
continue;
} }
addToMatchedTrees(parent);
} }
} else { } else {
// if (tree.getType() == 14) {
// ITree parent = getParentStatement(tree);
// if (parent == null) {
// matchTrees(tree.getChildren());
// } else {
// addToMatchedTrees(parent);
// }
// } else {
// matchTrees(tree.getChildren());
// }
matchTrees(tree.getChildren()); matchTrees(tree.getChildren());
} }
} }
} }
private void addToMatchedTrees(ITree tree) {
if (!matchedTrees.contains(tree)) {
/*
* TODO with the same parent, or the sub trees.
* In the same method body.
*/
matchedTrees.add(tree);
}
if (containsBlockStatement(tree)) {
int endLine = cUnit.getLineNumber(tree.getPos() + tree.getLength());
if (endLine > alarmEndLine) {
tree = removeBlock(tree);
}
}
}
private ITree removeBlock(ITree tree) { private ITree removeBlock(ITree tree) {
List<ITree> oldChildren = tree.getChildren(); List<ITree> oldChildren = tree.getChildren();
List<ITree> newChildren = new ArrayList<>(); List<ITree> newChildren = new ArrayList<>();
@@ -127,7 +196,7 @@ public class AlarmTree {
int endPosition = startPosition + child.getLength(); int endPosition = startPosition + child.getLength();
int endLine = cUnit.getLineNumber(endPosition); int endLine = cUnit.getLineNumber(endPosition);
if (endLine > this.alarmEndLine) { if (endLine > this.alarmEndLine) {
if (child.getType() == 8 || containsBlockStatement(child)) { if (child.getType() == 8 || containsBlockStatement(child)) { // 8: Block
child = removeBlock(child); child = removeBlock(child);
if (child.getChildren().size() == 0) { if (child.getChildren().size() == 0) {
continue; continue;
@@ -144,8 +213,12 @@ public class AlarmTree {
ITree parent = tree; ITree parent = tree;
do { do {
parent = parent.getParent(); parent = parent.getParent();
if (parent == null) {
return null;
}
int type = parent.getType(); int type = parent.getType();
if (type == 1 || type == 31 || type == 55) { if (type == 1 || type == 31 || type == 55 || type == 71) {
// AnonymousClassDeclaration // AnonymousClassDeclaration
// MethodDeclaration Initializer (type == 28) // MethodDeclaration Initializer (type == 28)
// TypeDeclaration // TypeDeclaration
src/main/java/edu/lu/uni/serval/violation/code/parser/ViolationCodeParser.java
+147
View File
@@ -0,0 +1,147 @@
package edu.lu.uni.serval.violation.code.parser;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.FixPatternParser.Tokenizer;
import edu.lu.uni.serval.FixPatternParser.violations.Violation;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
import edu.lu.uni.serval.utils.FileHelper;
public class ViolationCodeParser {
private static Logger log = LoggerFactory.getLogger(ViolationCodeParser.class);
public static void main(String[] args) {
/*
* Fixed violations:
* Violation code files.
* Position files.
*/
String fixedViolationFilesPath = Configuration.GUM_TREE_INPUT + "prevFiles/";
String positionsFilePath = Configuration.GUM_TREE_INPUT + "positions/";
int subIndex = 5;
String outputPath = Configuration.ROOT_PATH + "Alarms_tokens/fixedAlarms.list";
FileHelper.deleteFile(outputPath);
new ViolationCodeParser().parse(fixedViolationFilesPath, positionsFilePath, subIndex, outputPath);
/*
* UnFixed violations:
* Violation code files.
* Position files.
*/
String unfixedViolationFilesPath = Configuration.GUM_TREE_INPUT + "unfixAlarms/";
String un_positionsFilePath = Configuration.GUM_TREE_INPUT + "un_positions/";
subIndex = 8;
outputPath = Configuration.ROOT_PATH + "Alarms_tokens/unfixedAlarms.list";
FileHelper.deleteFile(outputPath);
new ViolationCodeParser().parse(unfixedViolationFilesPath, un_positionsFilePath, subIndex, outputPath);
}
public void parse(String alarmFilesPath, String positionFilesPath, int subIndex, String outputPath) {
StringBuilder tokensBuilder = new StringBuilder();
List<File> javaFiles = FileHelper.getAllFilesInCurrentDiectory(alarmFilesPath, ".java");
int counter = 0;
int a = 0;
int maxLength = 0;
for (File javaFile : javaFiles) {
String fileName = javaFile.getName().replace(".java", ".txt");
fileName = fileName.substring(subIndex);
List<Violation> violations = readViolationInfo(positionFilesPath + fileName);
for (Violation violation : violations) {
int startLine = violation.getStartLineNum();
int endLine = violation.getEndLineNum();
String alarmType = violation.getAlarmType();
// if (endLine > startLine + 5) {
// log.warn("#Large_Violation_Hunk: " + fileName.replace("#", "/").replace(".txt", ".java") + ":" + startLine + ":" + endLine + ":" + alarmType);
// continue;
// }
AlarmTree alarmTree = new AlarmTree(javaFile, startLine, endLine);
alarmTree.extract();
List<ITree> matchedTrees = alarmTree.getAlarmTrees();
if (matchedTrees.size() == 0) {
System.out.println(fileName + " == " + startLine + " : " + endLine);
a ++;
log.warn("#Null_Violation_Hunk: " + fileName.replace("#", "/").replace(".txt", ".java") + ":" + startLine + ":" + endLine + ":" + alarmType);
continue;
}
SimpleTree simpleTree = new SimpleTree();
simpleTree.setLabel("Block");
simpleTree.setNodeType("Block");
List<SimpleTree> children = new ArrayList<>();
for (ITree matchedTree : matchedTrees) {
SimpleTree simpleT = new SimplifyTree().canonicalizeSourceCodeTree(matchedTree, null);
children.add(simpleT);
}
simpleTree.setChildren(children);
String tokens = Tokenizer.getTokensDeepFirst(simpleTree);
String[] tokensArray = tokens.split(" ");
int length = tokensArray.length;
if (length > maxLength) maxLength = length;
tokensBuilder.append(alarmType + ":" + fileName + ":" + alarmTree.getAlarmFinalStartLine() + ":" + alarmTree.getAlarmFinalEndLine() + ":" + tokens + "\n");
counter ++;
if (counter % 5000 == 0) {
FileHelper.outputToFile(outputPath, tokensBuilder, true);
tokensBuilder.setLength(0);
}
}
}
System.out.println(counter);
System.out.println(a);
System.out.println("MaxLength: " + maxLength);
FileHelper.outputToFile(outputPath, tokensBuilder, true);
tokensBuilder.setLength(0);
}
private List<Violation> readViolationInfo(String file) {
List<Violation> violations = new ArrayList<>();
String fileContent = FileHelper.readFile(file);
BufferedReader reader = null;
reader = new BufferedReader(new StringReader(fileContent));
String line = null;
try {
while ((line = reader.readLine()) != null) {
String[] positionStr = line.split(":");
int startLine = Integer.parseInt(positionStr[1]);
int endLine = Integer.parseInt(positionStr[2]);
String alarmType = positionStr[0];
if (startLine == -1 || endLine == -1) {
log.warn("#Illegal_Line_Position: " + FileHelper.getFileName(file).replace("#", "/").replace(".txt", ".java") + ":" + startLine + ":" + endLine + ":" + alarmType);
continue;
}
Violation violation = new Violation(startLine, endLine, alarmType);
violations.add(violation);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return violations;
}
}