azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re-encapsulate the process of parsing patches.

Browse Source
This commit is contained in:
Kui LIU
2017-08-04 16:08:34 +02:00
parent cfc98f680b
commit 0a6892d7d4
14 changed files with 677 additions and 1500 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/FixPatternParser/HunkParser.java
-282
View File
@@ -1,282 +0,0 @@
package edu.lu.uni.serval.FixPatternParser;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import com.github.gumtreediff.actions.model.Action;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.diffentry.DiffEntryHunk;
import edu.lu.uni.serval.diffentry.DiffEntryReader;
import edu.lu.uni.serval.gumtree.GumTreeComparer;
import edu.lu.uni.serval.gumtree.regroup.ActionFilter;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper;
import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter;
import edu.lu.uni.serval.gumtree.regroup.HunkFixPattern;
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
/**
* Parse fix patterns with GumTree.
*
* @author kui.liu
*
*/
public class HunkParser {
private String astEditScripts = ""; // it will be used for fix patterns mining.
private String patchesSourceCode = ""; // testing
private String buggyTrees = ""; // Compute similarity for bug localization.
private String sizes = ""; // fix patterns' selection before mining.
private String tokensOfSourceCode = ""; // Compute similarity for bug localization.
private String originalTree = ""; // Guide of generating patches.
private String actionSets = ""; // Guide of generating patches.
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) {
// GumTree results
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
if (gumTreeResults != null && gumTreeResults.size() > 0) {
List<HierarchicalActionSet> actionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
ActionFilter filter = new ActionFilter();
// Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part.
// TODO: variable effects range, sub-actions are these kinds of modification?
List<HierarchicalActionSet> allActionSets = filter.filterOutUselessActions(actionSets);
if (allActionSets.size() == 0) return;
// DiffEntry size: filter out big hunks.
List<DiffEntryHunk> diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile);
//Filter out the modify actions, which are not in the DiffEntry hunks.
HunkActionFilter hunkFilter = new HunkActionFilter();
List<HunkFixPattern> allHunkFixPatternss = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, allActionSets, revFile, prevFile);
for (HunkFixPattern hunkFixPattern : allHunkFixPatternss) {
// Range of buggy source code
int startLine = 0;
int endLine = 0;
// Range of fixing source code
int startLine2 = 0;
int endLine2 = 0;
/*
* Convert the ITree of buggy code to a simple tree.
* It will be used to compute the similarity.
*/
List<HierarchicalActionSet> hunkActionSets = hunkFixPattern.getHunkActionSets();
SimpleTree simpleTree = new SimpleTree();
simpleTree.setLabel("Block");
simpleTree.setNodeType("Block");
List<SimpleTree> children = new ArrayList<>();
String astEditScripts = "";
for (HierarchicalActionSet hunkActionSet : hunkActionSets) {
SimplifyTree abstractIdentifier = new SimplifyTree();
abstractIdentifier.abstractTree(hunkActionSet);
SimpleTree simpleT = hunkActionSet.getSimpleTree();
if (simpleT == null) { // Failed to get the simple tree for INS actions.
continue;
}
children.add(simpleT);
/**
* Select edit scripts for deep learning.
* Edit scripts will be used to mine common fix patterns.
*/
// 1. First level: AST node type.
astEditScripts += getASTEditScripts(hunkActionSet);
// 2. source code: raw tokens
// 3. abstract identifiers:
// 4. semi-source code:
if (startLine == 0) {
startLine = hunkActionSet.getBugStartLineNum();
endLine = hunkActionSet.getBugEndLineNum();
startLine2 = hunkActionSet.getFixStartLineNum();
endLine2 = hunkActionSet.getFixEndLineNum();
} else {
if (startLine > hunkActionSet.getBugStartLineNum()) startLine = hunkActionSet.getBugStartLineNum();
if (startLine2 > hunkActionSet.getFixStartLineNum()) startLine2 = hunkActionSet.getFixStartLineNum();
if (endLine < hunkActionSet.getBugEndLineNum()) endLine = hunkActionSet.getBugEndLineNum();
if (endLine2 < hunkActionSet.getFixEndLineNum()) endLine2 = hunkActionSet.getFixEndLineNum();
}
}
if (children.size() == 0) continue;
if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue;
simpleTree.setChildren(children);
simpleTree.setParent(null);
// Source Code of patches.
String patchSourceCode = getPatchSourceCode(hunkFixPattern.getHunk(), startLine, endLine, startLine2, endLine2);
if ("".equals(patchSourceCode)) continue;
this.patchesSourceCode += "PATCH###\n" + patchSourceCode + "\n";
int size = astEditScripts.split(" ").length;
this.sizes += size + "\n";
this.astEditScripts += astEditScripts + "\n";
// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n";
this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n";
// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n";
// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n";
}
}
}
private String readActionSet(HierarchicalActionSet actionSet, String line) {
String str = line + actionSet.getActionString() + "\n";
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
for (HierarchicalActionSet subAction : subActions) {
str += readActionSet(subAction, line + "---");
}
return str;
}
private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private String getRawTokenEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private String getPatchSourceCode(DiffEntryHunk hunk, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) {
String sourceCode = hunk.getHunk();
int bugStartLine = hunk.getBugLineStartNum();
int fixStartLine = hunk.getFixLineStartNum();String buggyStatements = "";
String fixedStatements = "";
BufferedReader reader = null;
try {
reader = new BufferedReader(new StringReader(sourceCode));
String line = null;
int bugLines = 0;
int fixLines = 0;
int contextLines = 0; // counter of non-buggy code line.
while ((line = reader.readLine()) != null) {
int bugLineIndex = bugLines + contextLines;
int fixLineIndex = fixLines + contextLines;
if (line.startsWith("-")) {
if (bugStartLine + bugLineIndex >= startLineNum && bugStartLine + bugLineIndex <= endLineNum) {
buggyStatements += line + "\n";
}
bugLines ++;
} else if (line.startsWith("+")) {
if (fixStartLine + fixLineIndex >= startLineNum2 && fixStartLine + fixLineIndex <= endLineNum2) {
fixedStatements += line + "\n";
}
fixLines ++;
} else {
contextLines ++;
}
if (bugStartLine + bugLineIndex > endLineNum && fixStartLine + fixLineIndex > endLineNum2) {
break;
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (reader != null) {
reader.close();
reader = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
return buggyStatements + fixedStatements;
}
/**
* Get the AST node based edit script of patches in terms of breadth first.
*
* @param actionSet
* @return
*/
private String getASTEditScripts(HierarchicalActionSet actionSet) {
String editScript = "";
List<HierarchicalActionSet> actionSets = new ArrayList<>();
actionSets.add(actionSet);
while (actionSets.size() != 0) {
List<HierarchicalActionSet> subSets = new ArrayList<>();
for (HierarchicalActionSet set : actionSets) {
subSets.addAll(set.getSubActions());
String actionStr = set.getActionString();
int index = actionStr.indexOf("@@");
String singleEdit = actionStr.substring(0, index).replace(" ", "");
if (singleEdit.endsWith("SimpleName")) {
actionStr = actionStr.substring(index + 2);
if (actionStr.startsWith("MethodName")) {
singleEdit = singleEdit.replace("SimpleName", "MethodName");
} else {
if (actionStr.startsWith("Name")) {
actionStr = actionStr.substring(5, 6);
if (!actionStr.equals(actionStr.toLowerCase())) {
singleEdit = singleEdit.replace("SimpleName", "Name");
} else {
singleEdit = singleEdit.replace("SimpleName", "Variable");
}
} else {
singleEdit = singleEdit.replace("SimpleName", "Variable");
}
}
}
editScript += singleEdit + " ";
}
actionSets.clear();
actionSets.addAll(subSets);
}
return editScript;
}
private void clearITree(HierarchicalActionSet actionSet) {
actionSet.getAction().setNode(null);
for (HierarchicalActionSet subActionSet : actionSet.getSubActions()) {
clearITree(subActionSet);
}
}
public String getAstEditScripts() {
return astEditScripts;
}
public String getPatchesSourceCode() {
return patchesSourceCode;
}
public String getBuggyTrees() {
return buggyTrees;
}
public String getSizes() {
return sizes;
}
public String getTokensOfSourceCode() {
return tokensOfSourceCode;
}
public String getOriginalTree() {
return originalTree;
}
public String getActionSets() {
return actionSets;
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java
+77 -270
View File
@@ -1,26 +1,17 @@
package edu.lu.uni.serval.FixPatternParser;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.actions.model.Update;
import com.github.gumtreediff.actions.model.Move;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.diffentry.DiffEntryHunk;
import edu.lu.uni.serval.diffentry.DiffEntryReader;
import edu.lu.uni.serval.gumtree.GumTreeComparer;
import edu.lu.uni.serval.gumtree.regroup.ActionFilter;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper;
import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter;
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
/**
* Parse fix patterns with GumTree.
@@ -28,156 +19,78 @@ import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
* @author kui.liu
*
*/
public class Parser {
public abstract class Parser implements ParserInterface {
private String astEditScripts = ""; // it will be used for fix patterns mining.
private String patchesSourceCode = ""; // testing
private String buggyTrees = ""; // Compute similarity for bug localization.
private String sizes = ""; // fix patterns' selection before mining.
private String tokensOfSourceCode = ""; // Compute similarity for bug localization.
private String originalTree = ""; // Guide of generating patches.
private String actionSets = ""; // Guide of generating patches.
protected String astEditScripts = ""; // it will be used for fix patterns mining.
protected String patchesSourceCode = ""; // testing
protected String buggyTrees = ""; // Compute similarity for bug localization.
protected String sizes = ""; // fix patterns' selection before mining.
protected String tokensOfSourceCode = ""; // Compute similarity for bug localization.
protected String originalTree = ""; // Guide of generating patches.
protected String actionSets = ""; // Guide of generating patches.
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) {
public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile);
protected List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree(File prevFile, File revFile) {
List<HierarchicalActionSet> actionSets = new ArrayList<>();
// GumTree results
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
if (gumTreeResults != null && gumTreeResults.size() > 0) {
List<HierarchicalActionSet> actionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
// Regroup GumTre results.
List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
/**
* TODO What we need to discuss:
* 3. actions' nodes have the same parent belongs to one fix pattern?
* actions in the same method body.
* field, one by one,
* contains a body block.
*/
ActionFilter filter = new ActionFilter();
// Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part.
List<HierarchicalActionSet> hierarchicalActionSets = filter.filterOutUselessActions(actionSets); // TODO: variable effects range, sub-actions are these kinds of modification?
// DiffEntry size:
List<DiffEntryHunk> diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile); // filter out big hunks.
//Filter out the modify actions, which are not in the DiffEntry hunks.
HunkActionFilter hunkFilter = new HunkActionFilter();
hierarchicalActionSets = hunkFilter.filterActionsByDiffEntryHunk(diffentryHunks, actionSets, revFile, prevFile);
/**
* Patch size；
* 1. one hunk is one patch。
* 2. one statement.
*/
//
for (HierarchicalActionSet actionSet : hierarchicalActionSets) {
// position of buggy statements
int startPosition = 0;
int endPosition = 0;
// position of fixed statements
int startPosition2 = 0;
int endPosition2 = 0;
String actionStr = actionSet.getActionString();
String astNodeType = actionSet.getAstNodeType();
if (actionStr.startsWith("INS")) {
continue;
// startPosition2 = actionSet.getStartPosition();
// endPosition2 = startPosition2 + actionSet.getLength();
//
// if ("EnhancedForStatement".equals(astNodeType) || "ForStatement".equals(astNodeType)
// || "DoStatement".equals(astNodeType) || "WhileStatement".equals(astNodeType)
// || "LabeledStatement".equals(astNodeType) || "SynchronizedStatement".equals(astNodeType)
// || "IfStatement".equals(astNodeType) || "TryStatement".equals(astNodeType)) {
// List<Move> firstAndLastMov = getFirstAndLastMoveAction(actionSet);
// if (firstAndLastMov != null) {
// startPosition = firstAndLastMov.get(0).getNode().getPos();
// ITree lastTree = firstAndLastMov.get(1).getNode();
// endPosition = lastTree.getPos() + lastTree.getLength();
// } else { // Pure insert actions without any move actions.
// continue;
// }
// } else { // other insert statements
// continue;
// }
} else if (actionStr.startsWith("UPD")) {
startPosition = actionSet.getStartPosition();
endPosition = startPosition + actionSet.getLength();
Update update = (Update) actionSet.getAction();
ITree newNode = update.getNewNode();
startPosition2 = newNode.getPos();
endPosition2 = startPosition2 + newNode.getLength();
if ("EnhancedForStatement".equals(astNodeType) || "ForStatement".equals(astNodeType)
|| "DoStatement".equals(astNodeType) || "WhileStatement".equals(astNodeType)
|| "LabeledStatement".equals(astNodeType) || "SynchronizedStatement".equals(astNodeType)
|| "IfStatement".equals(astNodeType) || "TryStatement".equals(astNodeType)) {
List<ITree> children = update.getNode().getChildren();
endPosition = getEndPosition(children);
List<ITree> newChildren = newNode.getChildren();
endPosition2 = getEndPosition(newChildren);
if (endPosition == 0 || endPosition2 == 0) {
continue;
}
}
} else {// DEL actions and MOV actions: we don't need these actions, as for now.
continue;
}
// Get the buggy code and fixed code
if (startPosition != 0 && startPosition2 != 0) {
/*
* Convert the ITree of buggy code to a simple tree.
* It will be used to compute the similarity.
*/
SimplifyTree abstractIdentifier = new SimplifyTree();
abstractIdentifier.abstractTree(actionSet);
SimpleTree simpleTree = actionSet.getSimpleTree();
if (simpleTree == null) { // Failed to get the simple tree for INS actions.
continue;
}
/**
* Select edit scripts for deep learning.
* Edit scripts will be used to mine common fix patterns.
*/
// 1. First level: AST node type.
String astEditScripts = getASTEditScripts(actionSet);
int size = astEditScripts.split(" ").length;
if (size == 1) {
System.out.println(actionSet);
System.out.println(revFile.getPath());
// continue;
}
this.sizes += size + "\n";
this.astEditScripts += astEditScripts + "\n";
// 2. source code: raw tokens
String rawTokenEditScripts = getRawTokenEditScripts(actionSet);
// 3. abstract identifiers:
String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet);
// 4. semi-source code:
String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet);
this.buggyTrees += Configuration.BUGGY_TREE_SIGNAL + "\n" + simpleTree.toString() + "\n";
this.tokensOfSourceCode += getTokensDeepFirst(simpleTree).trim() + "\n";
this.actionSets += Configuration.BUGGY_TREE_SIGNAL + "\n" + readActionSet(actionSet, "") + "\n";
this.originalTree += Configuration.BUGGY_TREE_SIGNAL + "\n" + actionSet.getOriginalTree().toString() + "\n";
// // Source Code of patches.
// String patchSourceCode = getPatchSourceCode(sourceCode, startLineNum, endLineNum, startLineNum2,
// endLineNum2);
// if (patchSourceCode == null) continue;
// patchesSourceCode += "PATCH###\n" + patchSourceCode;
// patchesSourceCode += actionSet.toString() + "\n";
}
}
// TODO: variable effects range, sub-actions are these kinds of modification?
actionSets = new ActionFilter().filterOutUselessActions(allActionSets);
}
return actionSets;
}
private String readActionSet(HierarchicalActionSet actionSet, String line) {
protected List<Move> getFirstAndLastMoveAction(HierarchicalActionSet gumTreeResult) {
List<Move> firstAndLastMoveActions = new ArrayList<>();
List<HierarchicalActionSet> actions = gumTreeResult.getSubActions();
if (actions.size() == 0) {
return null;
}
Move firstMoveAction = null;
Move lastMoveAction = null;
while (actions.size() > 0) {
List<HierarchicalActionSet> subActions = new ArrayList<>();
for (HierarchicalActionSet action : actions) {
subActions.addAll(action.getSubActions());
if (action.toString().startsWith("MOV")) {
if (firstMoveAction == null) {
firstMoveAction = (Move) action.getAction();
lastMoveAction = (Move) action.getAction();
} else {
int startPosition = action.getStartPosition();
int length = action.getLength();
int startPositionFirst = firstMoveAction.getPosition();
int startPositionLast = lastMoveAction.getPosition();
int lengthLast = lastMoveAction.getNode().getLength();
if (startPosition < startPositionFirst || (startPosition == startPositionFirst && length > firstMoveAction.getLength())) {
firstMoveAction = (Move) action.getAction();
}
if ((startPosition + length) > (startPositionLast + lengthLast)) {
lastMoveAction = (Move) action.getAction();
}
}
}
}
actions.clear();
actions.addAll(subActions);
}
if (firstMoveAction == null) {
return null;
}
firstAndLastMoveActions.add(firstMoveAction);
firstAndLastMoveActions.add(lastMoveAction);
return firstAndLastMoveActions;
}
protected String readActionSet(HierarchicalActionSet actionSet, String line) {
String str = line + actionSet.getActionString() + "\n";
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
for (HierarchicalActionSet subAction : subActions) {
@@ -186,58 +99,22 @@ public class Parser {
return str;
}
private String getTokensDeepFirst(SimpleTree simpleTree) {
String tokens = "";
List<SimpleTree> children = simpleTree.getChildren();
String astNodeType = simpleTree.getNodeType();
if ("AssertStatement".equals(astNodeType) || "DoStatement".equals(astNodeType)
|| "ForStatement".equals(astNodeType) || "IfStatement".equals(astNodeType)
|| "ReturnStatement".equals(astNodeType) || "SwitchStatement".equals(astNodeType)
|| "SynchronizedStatement".equals(astNodeType) || "ThrowStatement".equals(astNodeType)
|| "TryStatement".equals(astNodeType) || "WhileStatement".equals(astNodeType)) {
String label = simpleTree.getLabel();
label = label.substring(0, label.indexOf("S")).toLowerCase();
tokens += label + " ";
} else if ("EnhancedForStatement".equals(astNodeType)) {
tokens += "for ";
} else if ("CatchClause".equals(astNodeType)) {
tokens += "catch ";
} else if ("SwitchCase".equals(astNodeType)) {
tokens += "case ";
} else if ("SuperConstructorInvocation".equals(astNodeType)) {
tokens += "super ";
} else if ("ConstructorInvocation".equals(astNodeType)) {
tokens += "this ";
} else if ("FinallyBody".equals(astNodeType)) {
tokens += "finally ";
}
if (children.isEmpty()) {
tokens += simpleTree.getNodeType() + " " + simpleTree.getLabel() + " ";
} else {
for (SimpleTree child : children) {
tokens += getTokensDeepFirst(child);
}
}
return tokens;
}
private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) {
protected String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) {
protected String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private String getRawTokenEditScripts(HierarchicalActionSet actionSet) {
protected String getRawTokenEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private int getEndPosition(List<ITree> children) {
protected int getEndPosition(List<ITree> children) {
int endPosition = 0;
for (ITree child : children) {
if (child.getLabel().endsWith("Body")) {
@@ -248,90 +125,13 @@ public class Parser {
return endPosition;
}
private String getPatchSourceCode(String sourceCode, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) {
String buggyStatements = "";
String fixedStatements = "";
BufferedReader reader = null;
try {
reader = new BufferedReader(new StringReader(sourceCode));
String line = null;
int startLine = 0;
int counter = 0;
int range = 0;
int startLine2 = 0;
int counter2 = 0;
int range2 = 0;
int counter3 = 0; // counter of non-buggy code line.
while ((line = reader.readLine()) != null) {
if (startLine == 0 && line.startsWith("@@ -")) {
// RegExp.filterSignal(line)
int plusIndex = line.indexOf("+");
String lineNum = line.substring(4, plusIndex);
String[] nums = lineNum.split(",");
if (nums.length != 2) {
continue;
}
startLine = Integer.parseInt(nums[0].trim());
range = Integer.parseInt(nums[1].trim());
if (startLine > endLineNum) {
return null; // Wrong Matching.
}
if (startLine + range < startLineNum) {
startLine = 0;
continue;
}
String lineNum2 = line.substring(plusIndex) .trim();
lineNum2 = lineNum2.substring(1, lineNum2.length() - 2);
String[] nums2 = lineNum2.split(",");
if (nums2.length != 2) {
startLine = 0;
range = 0;
continue;
}
startLine2 = Integer.parseInt(nums2[0].trim());
range2 = Integer.parseInt(nums2[1].trim());
continue;
}
int lineNum1 = counter + counter3;
int lineNum2 = counter2 + counter3;
if (startLine > 0 && startLine2 > 0 && lineNum1 < range && lineNum2 < range2) {
if (line.startsWith("-") && startLine + lineNum1 >= startLineNum && startLine + lineNum1 <= endLineNum) {
buggyStatements += line + "\n";
} else if (line.startsWith("+") && startLine2 + lineNum2 >= startLineNum2 && startLine2 + lineNum2 <= endLineNum2) {
fixedStatements += line + "\n";
}
if (line.startsWith("-")) {
counter ++;
} else if (line.startsWith("+")) {
counter2 ++;
} else {
counter3 ++;
}
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (reader != null) {
reader.close();
reader = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
return buggyStatements + fixedStatements;
}
/**
* Get the AST node based edit script of patches in terms of breadth first.
*
* @param actionSet
* @return
*/
private String getASTEditScripts(HierarchicalActionSet actionSet) {
protected String getASTEditScripts(HierarchicalActionSet actionSet) {
String editScript = "";
List<HierarchicalActionSet> actionSets = new ArrayList<>();
@@ -370,37 +170,44 @@ public class Parser {
return editScript;
}
private void clearITree(HierarchicalActionSet actionSet) {
protected void clearITree(HierarchicalActionSet actionSet) {
actionSet.getAction().setNode(null);
for (HierarchicalActionSet subActionSet : actionSet.getSubActions()) {
clearITree(subActionSet);
}
}
@Override
public String getAstEditScripts() {
return astEditScripts;
}
@Override
public String getPatchesSourceCode() {
return patchesSourceCode;
}
@Override
public String getBuggyTrees() {
return buggyTrees;
}
@Override
public String getSizes() {
return sizes;
}
@Override
public String getTokensOfSourceCode() {
return tokensOfSourceCode;
}
@Override
public String getOriginalTree() {
return originalTree;
}
@Override
public String getActionSets() {
return actionSets;
}
src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java
+23
View File
@@ -0,0 +1,23 @@
package edu.lu.uni.serval.FixPatternParser;
import java.io.File;
public interface ParserInterface {
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile);
public String getAstEditScripts();
public String getPatchesSourceCode();
public String getBuggyTrees();
public String getSizes();
public String getTokensOfSourceCode();
public String getOriginalTree();
public String getActionSets();
}
src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java
+2 -2
View File
@@ -7,9 +7,9 @@ public class RunnableParser implements Runnable {
private File prevFile;
private File revFile;
private File diffentryFile;
SingleStatementParser parser;
private Parser parser;
public RunnableParser(File prevFile, File revFile, File diffentryFile, SingleStatementParser parser) {
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) {
this.prevFile = prevFile;
this.revFile = revFile;
this.diffentryFile = diffentryFile;
src/main/java/edu/lu/uni/serval/FixPatternParser/SingleStatementParser.java
-401
View File
@@ -1,401 +0,0 @@
package edu.lu.uni.serval.FixPatternParser;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import org.eclipse.jdt.core.dom.CompilationUnit;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.actions.model.Move;
import com.github.gumtreediff.actions.model.Update;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.FixPattern.utils.Checker;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.diffentry.DiffEntryHunk;
import edu.lu.uni.serval.diffentry.DiffEntryReader;
import edu.lu.uni.serval.gumtree.GumTreeComparer;
import edu.lu.uni.serval.gumtree.regroup.ActionFilter;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper;
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
/**
* Parse fix patterns with GumTree.
*
* @author kui.liu
*
*/
public class SingleStatementParser {
private String astEditScripts = ""; // it will be used for fix patterns mining.
private String patchesSourceCode = ""; // testing
private String buggyTrees = ""; // Compute similarity for bug localization.
private String sizes = ""; // fix patterns' selection before mining.
private String tokensOfSourceCode = ""; // Compute similarity for bug localization.
private String originalTree = ""; // Guide of generating patches.
private String actionSets = ""; // Guide of generating patches.
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) {
// GumTree results
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
if (gumTreeResults != null && gumTreeResults.size() > 0) {
List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
// Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part.
// TODO: variable effects range, sub-actions are these kinds of modification?
List<HierarchicalActionSet> actionSets = new ActionFilter().filterOutUselessActions(allActionSets);
if (actionSets.size() > 0) {
// DiffEntry Hunks: filter out big hunks.
List<DiffEntryHunk> diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile);
for (HierarchicalActionSet actionSet : actionSets) {
// position of buggy statements
int startPosition = 0;
int endPosition = 0;
// position of fixed statements
int startPosition2 = 0;
int endPosition2 = 0;
String actionStr = actionSet.getActionString();
String astNodeType = actionSet.getAstNodeType();
if (actionStr.startsWith("INS")) {
startPosition2 = actionSet.getStartPosition();
endPosition2 = startPosition2 + actionSet.getLength();
List<Move> firstAndLastMov = getFirstAndLastMoveAction(actionSet);
if (firstAndLastMov != null) {
startPosition = firstAndLastMov.get(0).getNode().getPos();
ITree lastTree = firstAndLastMov.get(1).getNode();
endPosition = lastTree.getPos() + lastTree.getLength();
} else { // Ignore the pure insert actions without any move actions.
continue;
}
} else if (actionStr.startsWith("UPD")) {
startPosition = actionSet.getStartPosition();
endPosition = startPosition + actionSet.getLength();
Update update = (Update) actionSet.getAction();
ITree newNode = update.getNewNode();
startPosition2 = newNode.getPos();
endPosition2 = startPosition2 + newNode.getLength();
if (Checker.containsBodyBlock(astNodeType)) {
List<ITree> children = update.getNode().getChildren();
endPosition = getEndPosition(children);
List<ITree> newChildren = newNode.getChildren();
endPosition2 = getEndPosition(newChildren);
if (endPosition == 0) {
endPosition = startPosition + actionSet.getLength();
}
if (endPosition2 == 0) {
endPosition2 = startPosition2 + newNode.getLength();
}
}
} else {// DEL actions and MOV actions: we don't need these actions, as for now.
continue;
}
if (startPosition == 0 || startPosition2 == 0) {
continue;
}
CUCreator cuCreator = new CUCreator();
CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile);
CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile);
if (prevUnit == null || revUnit == null) {
continue;
}
// Get line numbers.
int startLine = prevUnit.getLineNumber(startPosition);
int endLine = prevUnit.getLineNumber(endPosition);
int startLine2 = revUnit.getLineNumber(startPosition2);
int endLine2 = revUnit.getLineNumber(endPosition2);
if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue;
//Filter out the modify actions, which are not in the DiffEntry hunks.
DiffEntryHunk hunk = matchHunk(startLine, endLine, startLine2, endLine2, actionStr, diffentryHunks);
if (hunk == null) {
continue;
}
/*
* Convert the ITree of buggy code to a simple tree.
* It will be used to compute the similarity.
*/
SimplifyTree abstractIdentifier = new SimplifyTree();
abstractIdentifier.abstractTree(actionSet);
SimpleTree simpleTree = actionSet.getSimpleTree();
if (simpleTree == null) { // Failed to get the simple tree for INS actions.
continue;
}
/**
* Select edit scripts for deep learning.
* Edit scripts will be used to mine common fix patterns.
*/
// 1. First level: AST node type.
String astEditScripts = getASTEditScripts(actionSet);
int size = astEditScripts.split(" ").length;
if (size == 1) {
// System.out.println(actionSet);
continue;
}
// Source Code of patches.
String patchSourceCode = getPatchSourceCode(hunk, startLine, endLine, startLine2, endLine2);
if ("".equals(patchSourceCode)) continue;
this.patchesSourceCode += Configuration.PATCH_SIGNAL + "\n" + patchSourceCode + "\n";
this.sizes += size + "\n";
this.astEditScripts += astEditScripts + "\n";
// 2. source code: raw tokens
String rawTokenEditScripts = getRawTokenEditScripts(actionSet);
// 3. abstract identifiers:
String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet);
// 4. semi-source code:
String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet);
// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n";
this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n";
// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n";
// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n";
}
actionSets.clear();
}
allActionSets.clear();
gumTreeResults.clear();
}
}
private DiffEntryHunk matchHunk(int startLine, int endLine, int startLine2, int endLine2, String actionStr, List<DiffEntryHunk> hunks) {
for (DiffEntryHunk hunk : hunks) {
int bugStartLine = hunk.getBugLineStartNum();
int bugRange = hunk.getBugRange();
int fixStartLine = hunk.getFixLineStartNum();
int fixRange = hunk.getFixRange();
if (actionStr.startsWith("INS")) {
if (fixStartLine + fixRange < startLine2) {
continue;
}
if (endLine2 < fixStartLine ) {
return null;
}
return hunk;
} else {
if (bugStartLine + bugRange < startLine) {
continue;
}
if (endLine < bugStartLine ) {
return null;
}
return hunk;
}
}
return null;
}
private List<Move> getFirstAndLastMoveAction(HierarchicalActionSet gumTreeResult) {
List<Move> firstAndLastMoveActions = new ArrayList<>();
List<HierarchicalActionSet> actions = gumTreeResult.getSubActions();
if (actions.size() == 0) {
return null;
}
Move firstMoveAction = null;
Move lastMoveAction = null;
while (actions.size() > 0) {
List<HierarchicalActionSet> subActions = new ArrayList<>();
for (HierarchicalActionSet action : actions) {
subActions.addAll(action.getSubActions());
if (action.toString().startsWith("MOV")) {
if (firstMoveAction == null) {
firstMoveAction = (Move) action.getAction();
lastMoveAction = (Move) action.getAction();
} else {
int startPosition = action.getStartPosition();
int length = action.getLength();
int startPositionFirst = firstMoveAction.getPosition();
int startPositionLast = lastMoveAction.getPosition();
int lengthLast = lastMoveAction.getNode().getLength();
if (startPosition < startPositionFirst || (startPosition == startPositionFirst && length > firstMoveAction.getLength())) {
firstMoveAction = (Move) action.getAction();
}
if ((startPosition + length) > (startPositionLast + lengthLast)) {
lastMoveAction = (Move) action.getAction();
}
}
}
}
actions.clear();
actions.addAll(subActions);
}
if (firstMoveAction == null) {
return null;
}
firstAndLastMoveActions.add(firstMoveAction);
firstAndLastMoveActions.add(lastMoveAction);
return firstAndLastMoveActions;
}
private String readActionSet(HierarchicalActionSet actionSet, String line) {
String str = line + actionSet.getActionString() + "\n";
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
for (HierarchicalActionSet subAction : subActions) {
str += readActionSet(subAction, line + "---");
}
return str;
}
private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private String getRawTokenEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private int getEndPosition(List<ITree> children) {
int endPosition = 0;
for (ITree child : children) {
if (child.getLabel().endsWith("Body")) {
endPosition = child.getPos() - 1;
break;
}
}
return endPosition;
}
private String getPatchSourceCode(DiffEntryHunk hunk, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) {
String sourceCode = hunk.getHunk();
int bugStartLine = hunk.getBugLineStartNum();
int fixStartLine = hunk.getFixLineStartNum();
String buggyStatements = "";
String fixedStatements = "";
BufferedReader reader = null;
try {
reader = new BufferedReader(new StringReader(sourceCode));
String line = null;
int bugLines = 0;
int fixLines = 0;
int contextLines = 0; // counter of non-buggy code line.
while ((line = reader.readLine()) != null) {
int bugLineIndex = bugLines + contextLines;
int fixLineIndex = fixLines + contextLines;
if (line.startsWith("-")) {
if (bugStartLine + bugLineIndex >= startLineNum && bugStartLine + bugLineIndex <= endLineNum) {
buggyStatements += line + "\n";
}
bugLines ++;
} else if (line.startsWith("+")) {
if (fixStartLine + fixLineIndex >= startLineNum2 && fixStartLine + fixLineIndex <= endLineNum2) {
fixedStatements += line + "\n";
}
fixLines ++;
} else {
contextLines ++;
}
if (bugStartLine + bugLineIndex > endLineNum && fixStartLine + fixLineIndex > endLineNum2) {
break;
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (reader != null) {
reader.close();
reader = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
return buggyStatements + fixedStatements;
}
/**
* Get the AST node based edit script of patches in terms of breadth first.
*
* @param actionSet
* @return
*/
private String getASTEditScripts(HierarchicalActionSet actionSet) {
String editScript = "";
List<HierarchicalActionSet> actionSets = new ArrayList<>();
actionSets.add(actionSet);
while (actionSets.size() != 0) {
List<HierarchicalActionSet> subSets = new ArrayList<>();
for (HierarchicalActionSet set : actionSets) {
subSets.addAll(set.getSubActions());
String actionStr = set.getActionString();
int index = actionStr.indexOf("@@");
String singleEdit = actionStr.substring(0, index).replace(" ", "");
if (singleEdit.endsWith("SimpleName")) {
actionStr = actionStr.substring(index + 2);
if (actionStr.startsWith("MethodName")) {
singleEdit = singleEdit.replace("SimpleName", "MethodName");
} else {
if (actionStr.startsWith("Name")) {
actionStr = actionStr.substring(5, 6);
if (!actionStr.equals(actionStr.toLowerCase())) {
singleEdit = singleEdit.replace("SimpleName", "Name");
} else {
singleEdit = singleEdit.replace("SimpleName", "Variable");
}
} else {
singleEdit = singleEdit.replace("SimpleName", "Variable");
}
}
}
editScript += singleEdit + " ";
}
actionSets.clear();
actionSets.addAll(subSets);
}
return editScript;
}
public String getAstEditScripts() {
return astEditScripts;
}
public String getPatchesSourceCode() {
return patchesSourceCode;
}
public String getBuggyTrees() {
return buggyTrees;
}
public String getSizes() {
return sizes;
}
public String getTokensOfSourceCode() {
return tokensOfSourceCode;
}
public String getOriginalTree() {
return originalTree;
}
public String getActionSets() {
return actionSets;
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/TestParser.java
-111
View File
@@ -1,111 +0,0 @@
package edu.lu.uni.serval.FixPatternParser;
import static java.lang.System.err;
import java.io.File;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import edu.lu.uni.serval.utils.FileHelper;
/**
* Test fix pattern parser.
*/
public class TestParser {
private static final long MILLIS_TO_WAIT = 60L;
public static void main(String[] args) {
String testDataPath = "../GumTreeInput/"; //DiffEntries prevFiles revFiles
File inputFileDirector = new File(testDataPath);
File[] files = inputFileDirector.listFiles(); // project folders
String outputPath = "../GumTreeOutput/";
StringBuilder astEditScripts = new StringBuilder();
// StringBuilder originalTrees = new StringBuilder();
// StringBuilder buggyTrees = new StringBuilder();
// StringBuilder actionSets = new StringBuilder();
StringBuilder tokens = new StringBuilder();
StringBuilder sizes = new StringBuilder();
StringBuilder patches = new StringBuilder();
for (File file : files) {
String projectFolder = file.getPath();
File revFileFolder = new File(projectFolder + "/revFiles/");// revised file folder
File[] revFiles = revFileFolder.listFiles();
int a = 0;
System.out.println(file.getPath());
String outputFile = outputPath + file.getName();
FileHelper.deleteDirectory(outputFile);
for (File revFile : revFiles) {
String fileName = revFile.getName();
if (fileName.endsWith(".java")) {
File prevFile = new File(projectFolder + "/prevFiles/prev_" + fileName);// previous file
File diffentryFile = new File(projectFolder + "/DiffEntries/" + fileName.replace(".java", ".txt")); // DiffEntry file
SingleStatementParser parser = new SingleStatementParser();
// HunkParser parser = new HunkParser();
final ExecutorService executor = Executors.newSingleThreadExecutor();
// schedule the work
final Future<?> future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser));
try {
// where we wait for task to complete
future.get(MILLIS_TO_WAIT, TimeUnit.SECONDS);
astEditScripts.append(parser.getAstEditScripts());
// originalTrees.append(parser.getOriginalTree());
// buggyTrees.append(parser.getBuggyTrees());
// actionSets.append(parser.getActionSets());
tokens.append(parser.getTokensOfSourceCode());
sizes.append(parser.getSizes());
patches.append(parser.getPatchesSourceCode());
a ++;
System.out.println(a);
if (a % 100 == 0) {
FileHelper.outputToFile(outputFile + "/EditScripts.list", astEditScripts, true);
FileHelper.outputToFile(outputFile + "/Tokens.list", tokens, true);
FileHelper.outputToFile(outputFile + "/Sizes.list", sizes, true);
FileHelper.outputToFile(outputFile + "/Patches.list", patches, true);
astEditScripts.setLength(0);
tokens.setLength(0);
sizes.setLength(0);
patches.setLength(0);
System.out.println(a);
}
} catch (TimeoutException e) {
err.println("task timed out");
future.cancel(true /* mayInterruptIfRunning */ );
} catch (InterruptedException e) {
err.println("task interrupted");
} catch (ExecutionException e) {
err.println("task aborted");
} finally {
executor.shutdownNow();
}
// parser.parseFixPatterns(prevFile, revFile, diffentryFile);
}
}
// break;
FileHelper.outputToFile(outputFile + "/EditScripts.list", astEditScripts, true);
// FileHelper.outputToFile(outputFile + "/OriginalTrees.list", originalTrees, false);
// FileHelper.outputToFile(outputFile + "/BuggyTrees.list", buggyTrees, false);
// FileHelper.outputToFile(outputFile + "/ActionSets.list", actionSets, false);
FileHelper.outputToFile(outputFile + "/Tokens.list", tokens, true);
FileHelper.outputToFile(outputFile + "/Sizes.list", sizes, true);
FileHelper.outputToFile(outputFile + "/Patches.list", patches, true);
// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/EditScripts.list", astEditScripts, false);
// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/OriginalTrees.list", originalTrees, false);
// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/BuggyTrees.list", buggyTrees, false);
// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/ActionSets.list", actionSets, false);
// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/Tokens.list", tokens, false);
// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/Sizes.list", sizes, false);
// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/Patches.list", patches, false);
}
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchHunkParser.java
+113
View File
@@ -0,0 +1,113 @@
package edu.lu.uni.serval.FixPatternParser.patch;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import edu.lu.uni.serval.FixPatternParser.Tokenizer;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.diffentry.DiffEntryHunk;
import edu.lu.uni.serval.diffentry.DiffEntryReader;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter;
import edu.lu.uni.serval.gumtree.regroup.HunkFixPattern;
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
/**
* Parse fix patterns with GumTree.
*
* Multiple statements bugs.
*
* @author kui.liu
*
*/
public class CommitPatchHunkParser extends CommitPatchParser {
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) {
// GumTree results
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree(prevFile, revFile);
if (actionSets.size() > 0) {
// DiffEntry size: filter out big hunks.
List<DiffEntryHunk> diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile);
//Filter out the modify actions, which are not in the DiffEntry hunks.
HunkActionFilter hunkFilter = new HunkActionFilter();
List<HunkFixPattern> allHunkFixPatternss = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, actionSets, revFile, prevFile);
for (HunkFixPattern hunkFixPattern : allHunkFixPatternss) {
// Range of buggy source code
int startLine = 0;
int endLine = 0;
// Range of fixing source code
int startLine2 = 0;
int endLine2 = 0;
/*
* Convert the ITree of buggy code to a simple tree.
* It will be used to compute the similarity.
*/
List<HierarchicalActionSet> hunkActionSets = hunkFixPattern.getHunkActionSets();
SimpleTree simpleTree = new SimpleTree();
simpleTree.setLabel("Block");
simpleTree.setNodeType("Block");
List<SimpleTree> children = new ArrayList<>();
String astEditScripts = "";
for (HierarchicalActionSet hunkActionSet : hunkActionSets) {
SimplifyTree abstractIdentifier = new SimplifyTree();
abstractIdentifier.abstractTree(hunkActionSet);
SimpleTree simpleT = hunkActionSet.getSimpleTree();
if (simpleT == null) { // Failed to get the simple tree for INS actions.
continue;
}
children.add(simpleT);
/**
* Select edit scripts for deep learning.
* Edit scripts will be used to mine common fix patterns.
*/
// 1. First level: AST node type.
astEditScripts += getASTEditScripts(hunkActionSet);
// 2. source code: raw tokens
// 3. abstract identifiers:
// 4. semi-source code:
if (startLine == 0) {
startLine = hunkActionSet.getBugStartLineNum();
endLine = hunkActionSet.getBugEndLineNum();
startLine2 = hunkActionSet.getFixStartLineNum();
endLine2 = hunkActionSet.getFixEndLineNum();
} else {
if (startLine > hunkActionSet.getBugStartLineNum()) startLine = hunkActionSet.getBugStartLineNum();
if (startLine2 > hunkActionSet.getFixStartLineNum()) startLine2 = hunkActionSet.getFixStartLineNum();
if (endLine < hunkActionSet.getBugEndLineNum()) endLine = hunkActionSet.getBugEndLineNum();
if (endLine2 < hunkActionSet.getFixEndLineNum()) endLine2 = hunkActionSet.getFixEndLineNum();
}
}
if (children.size() == 0) continue;
if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue;
simpleTree.setChildren(children);
simpleTree.setParent(null);
// Source Code of patches.
String patchSourceCode = getPatchSourceCode(hunkFixPattern.getHunk(), startLine, endLine, startLine2, endLine2);
if ("".equals(patchSourceCode)) continue;
this.patchesSourceCode += "PATCH###\n" + patchSourceCode + "\n";
int size = astEditScripts.split(" ").length;
this.sizes += size + "\n";
this.astEditScripts += astEditScripts + "\n";
// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n";
this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n";
// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n";
// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n";
}
}
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchParser.java
+101
View File
@@ -0,0 +1,101 @@
package edu.lu.uni.serval.FixPatternParser.patch;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.List;
import edu.lu.uni.serval.FixPatternParser.Parser;
import edu.lu.uni.serval.diffentry.DiffEntryHunk;
/**
* Parse fix patterns with GumTree.
*
* @author kui.liu
*
*/
public class CommitPatchParser extends Parser{
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) {
}
protected DiffEntryHunk matchHunk(int startLine, int endLine, int startLine2, int endLine2, String actionStr, List<DiffEntryHunk> hunks) {
for (DiffEntryHunk hunk : hunks) {
int bugStartLine = hunk.getBugLineStartNum();
int bugRange = hunk.getBugRange();
int fixStartLine = hunk.getFixLineStartNum();
int fixRange = hunk.getFixRange();
if (actionStr.startsWith("INS")) {
if (fixStartLine + fixRange < startLine2) {
continue;
}
if (endLine2 < fixStartLine ) {
return null;
}
return hunk;
} else {
if (bugStartLine + bugRange < startLine) {
continue;
}
if (endLine < bugStartLine ) {
return null;
}
return hunk;
}
}
return null;
}
protected String getPatchSourceCode(DiffEntryHunk hunk, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) {
String sourceCode = hunk.getHunk();
int bugStartLine = hunk.getBugLineStartNum();
int fixStartLine = hunk.getFixLineStartNum();
String buggyStatements = "";
String fixedStatements = "";
BufferedReader reader = null;
try {
reader = new BufferedReader(new StringReader(sourceCode));
String line = null;
int bugLines = 0;
int fixLines = 0;
int contextLines = 0; // counter of non-buggy code line.
while ((line = reader.readLine()) != null) {
int bugLineIndex = bugLines + contextLines;
int fixLineIndex = fixLines + contextLines;
if (line.startsWith("-")) {
if (bugStartLine + bugLineIndex >= startLineNum && bugStartLine + bugLineIndex <= endLineNum) {
buggyStatements += line + "\n";
}
bugLines ++;
} else if (line.startsWith("+")) {
if (fixStartLine + fixLineIndex >= startLineNum2 && fixStartLine + fixLineIndex <= endLineNum2) {
fixedStatements += line + "\n";
}
fixLines ++;
} else {
contextLines ++;
}
if (bugStartLine + bugLineIndex > endLineNum && fixStartLine + fixLineIndex > endLineNum2) {
break;
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (reader != null) {
reader.close();
reader = null;
}
} catch (IOException e) {
e.printStackTrace();
}
}
return buggyStatements + fixedStatements;
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchSingleStatementParser.java
+164
View File
@@ -0,0 +1,164 @@
package edu.lu.uni.serval.FixPatternParser.patch;
import java.io.File;
import java.util.List;
import org.eclipse.jdt.core.dom.CompilationUnit;
import com.github.gumtreediff.actions.model.Move;
import com.github.gumtreediff.actions.model.Update;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.FixPattern.utils.Checker;
import edu.lu.uni.serval.FixPatternParser.CUCreator;
import edu.lu.uni.serval.FixPatternParser.Tokenizer;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.diffentry.DiffEntryHunk;
import edu.lu.uni.serval.diffentry.DiffEntryReader;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
/**
* Parse fix patterns with GumTree.
*
* Single Statement bugs.
*
* @author kui.liu
*
*/
public class CommitPatchSingleStatementParser extends CommitPatchParser {
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) {
// GumTree results
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree(prevFile, revFile);
if (actionSets.size() > 0) {
// DiffEntry Hunks: filter out big hunks.
List<DiffEntryHunk> diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile);
for (HierarchicalActionSet actionSet : actionSets) {
// position of buggy statements
int startPosition = 0;
int endPosition = 0;
// position of fixed statements
int startPosition2 = 0;
int endPosition2 = 0;
String actionStr = actionSet.getActionString();
String astNodeType = actionSet.getAstNodeType();
if (actionStr.startsWith("INS")) {
startPosition2 = actionSet.getStartPosition();
endPosition2 = startPosition2 + actionSet.getLength();
List<Move> firstAndLastMov = getFirstAndLastMoveAction(actionSet);
if (firstAndLastMov != null) {
startPosition = firstAndLastMov.get(0).getNode().getPos();
ITree lastTree = firstAndLastMov.get(1).getNode();
endPosition = lastTree.getPos() + lastTree.getLength();
} else { // Ignore the pure insert actions without any move
// actions.
continue;
}
} else if (actionStr.startsWith("UPD")) {
startPosition = actionSet.getStartPosition();
endPosition = startPosition + actionSet.getLength();
Update update = (Update) actionSet.getAction();
ITree newNode = update.getNewNode();
startPosition2 = newNode.getPos();
endPosition2 = startPosition2 + newNode.getLength();
if (Checker.containsBodyBlock(astNodeType)) {
List<ITree> children = update.getNode().getChildren();
endPosition = getEndPosition(children);
List<ITree> newChildren = newNode.getChildren();
endPosition2 = getEndPosition(newChildren);
if (endPosition == 0) {
endPosition = startPosition + actionSet.getLength();
}
if (endPosition2 == 0) {
endPosition2 = startPosition2 + newNode.getLength();
}
}
} else {// DEL actions and MOV actions: we don't need these
// actions, as for now.
continue;
}
if (startPosition == 0 || startPosition2 == 0) {
continue;
}
CUCreator cuCreator = new CUCreator();
CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile);
CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile);
if (prevUnit == null || revUnit == null) {
continue;
}
// Get line numbers.
int startLine = prevUnit.getLineNumber(startPosition);
int endLine = prevUnit.getLineNumber(endPosition);
int startLine2 = revUnit.getLineNumber(startPosition2);
int endLine2 = revUnit.getLineNumber(endPosition2);
if (endLine - startLine >= Configuration.HUNK_SIZE - 2
|| endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2)
continue;
// Filter out the modify actions, which are not in the DiffEntry
// hunks.
DiffEntryHunk hunk = matchHunk(startLine, endLine, startLine2, endLine2, actionStr, diffentryHunks);
if (hunk == null) {
continue;
}
/*
* Convert the ITree of buggy code to a simple tree. It will be
* used to compute the similarity.
*/
SimplifyTree abstractIdentifier = new SimplifyTree();
abstractIdentifier.abstractTree(actionSet);
SimpleTree simpleTree = actionSet.getSimpleTree();
if (simpleTree == null) { // Failed to get the simple tree for
// INS actions.
continue;
}
/**
* Select edit scripts for deep learning. Edit scripts will be
* used to mine common fix patterns.
*/
// 1. First level: AST node type.
String astEditScripts = getASTEditScripts(actionSet);
int size = astEditScripts.split(" ").length;
if (size == 1) {
// System.out.println(actionSet);
continue;
}
// Source Code of patches.
String patchSourceCode = getPatchSourceCode(hunk, startLine, endLine, startLine2, endLine2);
if ("".equals(patchSourceCode))
continue;
this.patchesSourceCode += Configuration.PATCH_SIGNAL + "\n" + patchSourceCode + "\n";
this.sizes += size + "\n";
this.astEditScripts += astEditScripts + "\n";
// 2. source code: raw tokens
String rawTokenEditScripts = getRawTokenEditScripts(actionSet);
// 3. abstract identifiers:
String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet);
// 4. semi-source code:
String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet);
// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" +
// simpleTree.toString() + "\n";
this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n";
// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" +
// readActionSet(actionSet, "") + "\n";
// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" +
// actionSet.getOriginalTree().toString() + "\n";
}
actionSets.clear();
}
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java
+24 -126
View File
@@ -6,21 +6,19 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import com.github.gumtreediff.actions.model.Action;
import java.util.Map;
import edu.lu.uni.serval.FixPatternParser.Tokenizer;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.diffentry.DiffEntryHunk;
import edu.lu.uni.serval.diffentry.DiffEntryReader;
import edu.lu.uni.serval.gumtree.GumTreeComparer;
import edu.lu.uni.serval.gumtree.regroup.ActionFilter;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper;
import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter;
import edu.lu.uni.serval.gumtree.regroup.HunkFixPattern;
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
import edu.lu.uni.serval.utils.MapSorter;
/**
* Parse fix patterns with GumTree.
@@ -28,34 +26,37 @@ import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
* @author kui.liu
*
*/
public class FixedViolationHunkParser {
public class FixedViolationHunkParser extends FixedViolationParser {
private String astEditScripts = ""; // it will be used for fix patterns mining.
private String patchesSourceCode = ""; // testing
private String buggyTrees = ""; // Compute similarity for bug localization.
private String sizes = ""; // fix patterns' selection before mining.
private String tokensOfSourceCode = ""; // Compute similarity for bug localization.
private String originalTree = ""; // Guide of generating patches.
private String actionSets = ""; // Guide of generating patches.
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) {
@Override
public void parseFixPatterns(File prevFile, File revFile, File positionsFile) {
// GumTree results
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree(prevFile, revFile);
if (gumTreeResults != null && gumTreeResults.size() > 0) {
List<HierarchicalActionSet> actionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
if (actionSets.size() > 0) {
Map<Integer, Integer> positions = readPositions(positionsFile);
if (positions.size() > 1) {
MapSorter<Integer, Integer> sorter = new MapSorter<>();
positions = sorter.sortByKeyAscending(positions);
}
for (Map.Entry<Integer, Integer> entry : positions.entrySet()) {
// only statements
for (HierarchicalActionSet actionSet : actionSets) {
String astNodeType = actionSet.getAstNodeType();
if (astNodeType.endsWith("Statement") || "FieldDeclaration".equals(astNodeType)) {
}
}
}
ActionFilter filter = new ActionFilter();
// Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part.
// TODO: variable effects range, sub-actions are these kinds of modification?
List<HierarchicalActionSet> allActionSets = filter.filterOutUselessActions(actionSets);
if (allActionSets.size() == 0) return;
// DiffEntry size: filter out big hunks.
List<DiffEntryHunk> diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile);
List<DiffEntryHunk> diffentryHunks = new DiffEntryReader().readHunks(positionsFile);
//Filter out the modify actions, which are not in the DiffEntry hunks.
HunkActionFilter hunkFilter = new HunkActionFilter();
List<HunkFixPattern> allHunkFixPatternss = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, allActionSets, revFile, prevFile);
List<HunkFixPattern> allHunkFixPatternss = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, actionSets, revFile, prevFile);
for (HunkFixPattern hunkFixPattern : allHunkFixPatternss) {
// Range of buggy source code
@@ -129,30 +130,6 @@ public class FixedViolationHunkParser {
}
}
private String readActionSet(HierarchicalActionSet actionSet, String line) {
String str = line + actionSet.getActionString() + "\n";
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
for (HierarchicalActionSet subAction : subActions) {
str += readActionSet(subAction, line + "---");
}
return str;
}
private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private String getRawTokenEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private String getPatchSourceCode(DiffEntryHunk hunk, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) {
String sourceCode = hunk.getHunk();
int bugStartLine = hunk.getBugLineStartNum();
@@ -201,83 +178,4 @@ public class FixedViolationHunkParser {
return buggyStatements + fixedStatements;
}
/**
* Get the AST node based edit script of patches in terms of breadth first.
*
* @param actionSet
* @return
*/
private String getASTEditScripts(HierarchicalActionSet actionSet) {
String editScript = "";
List<HierarchicalActionSet> actionSets = new ArrayList<>();
actionSets.add(actionSet);
while (actionSets.size() != 0) {
List<HierarchicalActionSet> subSets = new ArrayList<>();
for (HierarchicalActionSet set : actionSets) {
subSets.addAll(set.getSubActions());
String actionStr = set.getActionString();
int index = actionStr.indexOf("@@");
String singleEdit = actionStr.substring(0, index).replace(" ", "");
if (singleEdit.endsWith("SimpleName")) {
actionStr = actionStr.substring(index + 2);
if (actionStr.startsWith("MethodName")) {
singleEdit = singleEdit.replace("SimpleName", "MethodName");
} else {
if (actionStr.startsWith("Name")) {
actionStr = actionStr.substring(5, 6);
if (!actionStr.equals(actionStr.toLowerCase())) {
singleEdit = singleEdit.replace("SimpleName", "Name");
} else {
singleEdit = singleEdit.replace("SimpleName", "Variable");
}
} else {
singleEdit = singleEdit.replace("SimpleName", "Variable");
}
}
}
editScript += singleEdit + " ";
}
actionSets.clear();
actionSets.addAll(subSets);
}
return editScript;
}
private void clearITree(HierarchicalActionSet actionSet) {
actionSet.getAction().setNode(null);
for (HierarchicalActionSet subActionSet : actionSet.getSubActions()) {
clearITree(subActionSet);
}
}
public String getAstEditScripts() {
return astEditScripts;
}
public String getPatchesSourceCode() {
return patchesSourceCode;
}
public String getBuggyTrees() {
return buggyTrees;
}
public String getSizes() {
return sizes;
}
public String getTokensOfSourceCode() {
return tokensOfSourceCode;
}
public String getOriginalTree() {
return originalTree;
}
public String getActionSets() {
return actionSets;
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java
+7 -304
View File
@@ -4,28 +4,10 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.eclipse.jdt.core.dom.CompilationUnit;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.actions.model.Move;
import com.github.gumtreediff.actions.model.Update;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.FixPattern.utils.Checker;
import edu.lu.uni.serval.FixPatternParser.CUCreator;
import edu.lu.uni.serval.FixPatternParser.Tokenizer;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.gumtree.GumTreeComparer;
import edu.lu.uni.serval.gumtree.regroup.ActionFilter;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper;
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
import edu.lu.uni.serval.FixPatternParser.Parser;
import edu.lu.uni.serval.utils.FileHelper;
/**
@@ -34,143 +16,13 @@ import edu.lu.uni.serval.utils.FileHelper;
* @author kui.liu
*
*/
public class FixedViolationParser {
public class FixedViolationParser extends Parser {
private String astEditScripts = ""; // it will be used for fix patterns mining.
private String patchesSourceCode = ""; // testing
private String buggyTrees = ""; // Compute similarity for bug localization.
private String sizes = ""; // fix patterns' selection before mining.
private String tokensOfSourceCode = ""; // Compute similarity for bug localization.
private String originalTree = ""; // Guide of generating patches.
private String actionSets = ""; // Guide of generating patches.
@Override
public void parseFixPatterns(File prevFile, File revFile, File positionFile) {
// GumTree results
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
if (gumTreeResults != null && gumTreeResults.size() > 0) {
List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
// Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part.
// TODO: variable effects range, sub-actions are these kinds of modification?
List<HierarchicalActionSet> actionSets = new ActionFilter().filterOutUselessActions(allActionSets);
if (actionSets.size() > 0) {
// DiffEntry Hunks: filter out big hunks.
Map<Integer, Integer> positions = readPositions(positionFile);
for (HierarchicalActionSet actionSet : actionSets) {
// position of buggy statements
int startPosition = 0;
int endPosition = 0;
// position of fixed statements
int startPosition2 = 0;
int endPosition2 = 0;
String actionStr = actionSet.getActionString();
String astNodeType = actionSet.getAstNodeType();
if (actionStr.startsWith("INS")) {
startPosition2 = actionSet.getStartPosition();
endPosition2 = startPosition2 + actionSet.getLength();
List<Move> firstAndLastMov = getFirstAndLastMoveAction(actionSet);
if (firstAndLastMov != null) {
startPosition = firstAndLastMov.get(0).getNode().getPos();
ITree lastTree = firstAndLastMov.get(1).getNode();
endPosition = lastTree.getPos() + lastTree.getLength();
} else { // Ignore the pure insert actions without any move actions.
continue;
}
} else if (actionStr.startsWith("UPD")) {
startPosition = actionSet.getStartPosition();
endPosition = startPosition + actionSet.getLength();
Update update = (Update) actionSet.getAction();
ITree newNode = update.getNewNode();
startPosition2 = newNode.getPos();
endPosition2 = startPosition2 + newNode.getLength();
if (Checker.containsBodyBlock(astNodeType)) {
List<ITree> children = update.getNode().getChildren();
endPosition = getEndPosition(children);
List<ITree> newChildren = newNode.getChildren();
endPosition2 = getEndPosition(newChildren);
if (endPosition == 0) {
endPosition = startPosition + actionSet.getLength();
}
if (endPosition2 == 0) {
endPosition2 = startPosition2 + newNode.getLength();
}
}
} else {// DEL actions and MOV actions: we don't need these actions, as for now.
continue;
}
if (startPosition == 0 || startPosition2 == 0) {
continue;
}
CUCreator cuCreator = new CUCreator();
CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile);
CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile);
if (prevUnit == null || revUnit == null) {
continue;
}
// Get line numbers.
int startLine = prevUnit.getLineNumber(startPosition);
int endLine = prevUnit.getLineNumber(endPosition);
int startLine2 = revUnit.getLineNumber(startPosition2);
int endLine2 = revUnit.getLineNumber(endPosition2);
if (!inPositions(startLine, endLine, positions)) {
continue;
}
// if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue;
/*
* Convert the ITree of buggy code to a simple tree.
* It will be used to compute the similarity.
*/
SimplifyTree abstractIdentifier = new SimplifyTree();
abstractIdentifier.abstractTree(actionSet);
SimpleTree simpleTree = actionSet.getSimpleTree();
if (simpleTree == null) { // Failed to get the simple tree for INS actions.
continue;
}
/**
* Select edit scripts for deep learning.
* Edit scripts will be used to mine common fix patterns.
*/
// 1. First level: AST node type.
String astEditScripts = getASTEditScripts(actionSet);
int size = astEditScripts.split(" ").length;
if (size == 1) {
continue;
}
// Source Code of patches.
String patchSourceCode = getPatchSourceCode(prevFile, revFile, startLine, endLine, startLine2, endLine2);
this.patchesSourceCode += Configuration.PATCH_SIGNAL + "\n" + revFile.getName() + "\n" + patchSourceCode + "\n";
this.sizes += size + "\n";
this.astEditScripts += astEditScripts + "\n";
// 2. source code: raw tokens
String rawTokenEditScripts = getRawTokenEditScripts(actionSet);
// 3. abstract identifiers:
String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet);
// 4. semi-source code:
String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet);
// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n";
this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n";
// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n";
// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n";
}
actionSets.clear();
}
allActionSets.clear();
gumTreeResults.clear();
}
}
private boolean inPositions(int startLine, int endLine, Map<Integer, Integer> positions) {
protected boolean inPositions(int startLine, int endLine, Map<Integer, Integer> positions) {
for (Map.Entry<Integer, Integer> entry : positions.entrySet()) {
int startPosi = entry.getKey();
int endPosi = entry.getValue();
@@ -179,7 +31,7 @@ public class FixedViolationParser {
return false;
}
private Map<Integer, Integer> readPositions(File positionFile) {
protected Map<Integer, Integer> readPositions(File positionFile) {
Map<Integer, Integer> positions = new HashMap<>();
String fileContent = FileHelper.readFile(positionFile);
BufferedReader reader = null;
@@ -204,91 +56,14 @@ public class FixedViolationParser {
return positions;
}
private List<Move> getFirstAndLastMoveAction(HierarchicalActionSet gumTreeResult) {
List<Move> firstAndLastMoveActions = new ArrayList<>();
List<HierarchicalActionSet> actions = gumTreeResult.getSubActions();
if (actions.size() == 0) {
return null;
}
Move firstMoveAction = null;
Move lastMoveAction = null;
while (actions.size() > 0) {
List<HierarchicalActionSet> subActions = new ArrayList<>();
for (HierarchicalActionSet action : actions) {
subActions.addAll(action.getSubActions());
if (action.toString().startsWith("MOV")) {
if (firstMoveAction == null) {
firstMoveAction = (Move) action.getAction();
lastMoveAction = (Move) action.getAction();
} else {
int startPosition = action.getStartPosition();
int length = action.getLength();
int startPositionFirst = firstMoveAction.getPosition();
int startPositionLast = lastMoveAction.getPosition();
int lengthLast = lastMoveAction.getNode().getLength();
if (startPosition < startPositionFirst || (startPosition == startPositionFirst && length > firstMoveAction.getLength())) {
firstMoveAction = (Move) action.getAction();
}
if ((startPosition + length) > (startPositionLast + lengthLast)) {
lastMoveAction = (Move) action.getAction();
}
}
}
}
actions.clear();
actions.addAll(subActions);
}
if (firstMoveAction == null) {
return null;
}
firstAndLastMoveActions.add(firstMoveAction);
firstAndLastMoveActions.add(lastMoveAction);
return firstAndLastMoveActions;
}
private String readActionSet(HierarchicalActionSet actionSet, String line) {
String str = line + actionSet.getActionString() + "\n";
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
for (HierarchicalActionSet subAction : subActions) {
str += readActionSet(subAction, line + "---");
}
return str;
}
private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private String getRawTokenEditScripts(HierarchicalActionSet actionSet) {
// TODO Auto-generated method stub
return null;
}
private int getEndPosition(List<ITree> children) {
int endPosition = 0;
for (ITree child : children) {
if (child.getLabel().endsWith("Body")) {
endPosition = child.getPos() - 1;
break;
}
}
return endPosition;
}
private String getPatchSourceCode(File prevFile, File revFile, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) {
protected String getPatchSourceCode(File prevFile, File revFile, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) {
String buggyStatements = readSourceCode(prevFile, startLineNum, endLineNum, "-");
String fixedStatements = readSourceCode(revFile, startLineNum2, endLineNum2, "+");
return buggyStatements + fixedStatements;
}
private String readSourceCode(File file, int startLineNum, int endLineNum, String type) {
protected String readSourceCode(File file, int startLineNum, int endLineNum, String type) {
String sourceCode = "";
String fileContent = FileHelper.readFile(file);
BufferedReader reader = null;
@@ -315,76 +90,4 @@ public class FixedViolationParser {
return sourceCode;
}
/**
* Get the AST node based edit script of patches in terms of breadth first.
*
* @param actionSet
* @return
*/
private String getASTEditScripts(HierarchicalActionSet actionSet) {
String editScript = "";
List<HierarchicalActionSet> actionSets = new ArrayList<>();
actionSets.add(actionSet);
while (actionSets.size() != 0) {
List<HierarchicalActionSet> subSets = new ArrayList<>();
for (HierarchicalActionSet set : actionSets) {
subSets.addAll(set.getSubActions());
String actionStr = set.getActionString();
int index = actionStr.indexOf("@@");
String singleEdit = actionStr.substring(0, index).replace(" ", "");
if (singleEdit.endsWith("SimpleName")) {
actionStr = actionStr.substring(index + 2);
if (actionStr.startsWith("MethodName")) {
singleEdit = singleEdit.replace("SimpleName", "MethodName");
} else {
if (actionStr.startsWith("Name")) {
actionStr = actionStr.substring(5, 6);
if (!actionStr.equals(actionStr.toLowerCase())) {
singleEdit = singleEdit.replace("SimpleName", "Name");
} else {
singleEdit = singleEdit.replace("SimpleName", "Variable");
}
} else {
singleEdit = singleEdit.replace("SimpleName", "Variable");
}
}
}
editScript += singleEdit + " ";
}
actionSets.clear();
actionSets.addAll(subSets);
}
return editScript;
}
public String getAstEditScripts() {
return astEditScripts;
}
public String getPatchesSourceCode() {
return patchesSourceCode;
}
public String getBuggyTrees() {
return buggyTrees;
}
public String getSizes() {
return sizes;
}
public String getTokensOfSourceCode() {
return tokensOfSourceCode;
}
public String getOriginalTree() {
return originalTree;
}
public String getActionSets() {
return actionSets;
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java
+158
View File
@@ -0,0 +1,158 @@
package edu.lu.uni.serval.FixPatternParser.violations;
import java.io.File;
import java.util.List;
import java.util.Map;
import org.eclipse.jdt.core.dom.CompilationUnit;
import com.github.gumtreediff.actions.model.Move;
import com.github.gumtreediff.actions.model.Update;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.FixPattern.utils.Checker;
import edu.lu.uni.serval.FixPatternParser.CUCreator;
import edu.lu.uni.serval.FixPatternParser.Tokenizer;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.SimpleTree;
import edu.lu.uni.serval.gumtree.regroup.SimplifyTree;
/**
* Parse fix patterns with GumTree.
*
* @author kui.liu
*
*/
public class FixedViolationSingleStatementParser extends FixedViolationParser {
@Override
public void parseFixPatterns(File prevFile, File revFile, File positionFile) {
// GumTree results
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree(prevFile, revFile);
if (actionSets.size() > 0) {
// DiffEntry Hunks: filter out big hunks.
Map<Integer, Integer> positions = readPositions(positionFile);
for (HierarchicalActionSet actionSet : actionSets) {
// position of buggy statements
int startPosition = 0;
int endPosition = 0;
// position of fixed statements
int startPosition2 = 0;
int endPosition2 = 0;
String actionStr = actionSet.getActionString();
String astNodeType = actionSet.getAstNodeType();
if (actionStr.startsWith("INS")) {
startPosition2 = actionSet.getStartPosition();
endPosition2 = startPosition2 + actionSet.getLength();
List<Move> firstAndLastMov = getFirstAndLastMoveAction(actionSet);
if (firstAndLastMov != null) {
startPosition = firstAndLastMov.get(0).getNode().getPos();
ITree lastTree = firstAndLastMov.get(1).getNode();
endPosition = lastTree.getPos() + lastTree.getLength();
} else { // Ignore the pure insert actions without any move actions.
continue;
}
} else if (actionStr.startsWith("UPD")) {
startPosition = actionSet.getStartPosition();
endPosition = startPosition + actionSet.getLength();
Update update = (Update) actionSet.getAction();
ITree newNode = update.getNewNode();
startPosition2 = newNode.getPos();
endPosition2 = startPosition2 + newNode.getLength();
if (Checker.containsBodyBlock(astNodeType)) {
List<ITree> children = update.getNode().getChildren();
endPosition = getEndPosition(children);
List<ITree> newChildren = newNode.getChildren();
endPosition2 = getEndPosition(newChildren);
if (endPosition == 0) {
endPosition = startPosition + actionSet.getLength();
}
if (endPosition2 == 0) {
endPosition2 = startPosition2 + newNode.getLength();
}
}
} else if (actionStr.startsWith("MOV")) {// DEL actions and MOV actions: we don't need these actions, as for now.
continue;
// startPosition = actionSet.getStartPosition();
// endPosition = startPosition + actionSet.getLength();
// ITree node = actionSet.getNode().getParent();
// startPosition2 = node.getPos();
// endPosition2 = startPosition2 + node.getLength();
} else {
startPosition = actionSet.getStartPosition();
endPosition = startPosition + actionSet.getLength();
continue;
}
if (startPosition == 0 || startPosition2 == 0) {
continue;
}
CUCreator cuCreator = new CUCreator();
CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile);
CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile);
if (prevUnit == null || revUnit == null) {
continue;
}
// Get line numbers.
int startLine = prevUnit.getLineNumber(startPosition);
int endLine = prevUnit.getLineNumber(endPosition);
int startLine2 = revUnit.getLineNumber(startPosition2);
int endLine2 = revUnit.getLineNumber(endPosition2);
if (!inPositions(startLine, endLine, positions)) {
continue;
}
if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue;
/*
* Convert the ITree of buggy code to a simple tree.
* It will be used to compute the similarity.
*/
SimplifyTree abstractIdentifier = new SimplifyTree();
abstractIdentifier.abstractTree(actionSet);
SimpleTree simpleTree = actionSet.getSimpleTree();
if (simpleTree == null) { // Failed to get the simple tree for INS actions.
continue;
}
/**
* Select edit scripts for deep learning.
* Edit scripts will be used to mine common fix patterns.
*/
// 1. First level: AST node type.
String astEditScripts = getASTEditScripts(actionSet);
int size = astEditScripts.split(" ").length;
if (size == 1) {
continue;
}
// Source Code of patches.
String patchSourceCode = getPatchSourceCode(prevFile, revFile, startLine, endLine, startLine2, endLine2);
this.patchesSourceCode += Configuration.PATCH_SIGNAL + "\n" + revFile.getName() + "\n" + patchSourceCode + "\n";
this.sizes += size + "\n";
this.astEditScripts += astEditScripts + "\n";
// 2. source code: raw tokens
String rawTokenEditScripts = getRawTokenEditScripts(actionSet);
// 3. abstract identifiers:
String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet);
// 4. semi-source code:
String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet);
// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n";
this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n";
// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n";
// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n";
}
actionSets.clear();
} else {
System.out.println(1);
}
}
}
src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java
+2 -2
View File
@@ -18,7 +18,7 @@ import akka.actor.Props;
import akka.actor.UntypedActor;
import akka.japi.Creator;
import edu.lu.uni.serval.FixPatternParser.RunnableParser;
import edu.lu.uni.serval.FixPatternParser.SingleStatementParser;
import edu.lu.uni.serval.FixPatternParser.patch.CommitPatchParser;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
@@ -67,7 +67,7 @@ public class ParseFixPatternWorker extends UntypedActor {
File revFile = msgFile.getRevFile();
File prevFile = msgFile.getPrevFile();
File diffentryFile = msgFile.getDiffEntryFile();
SingleStatementParser parser = new SingleStatementParser();
CommitPatchParser parser = new CommitPatchParser();
final ExecutorService executor = Executors.newSingleThreadExecutor();
// schedule the work
src/main/java/edu/lu/uni/serval/violation/parse/ViolationParser.java
+6 -2
View File
@@ -47,7 +47,7 @@ public class ViolationParser {
public void parseViolations(String fixedAlarmFile, List<File> repos) {
AlarmsReader reader = new AlarmsReader();
Map<String, Violation> violations = reader.readAlarmsList(fixedAlarmFile);
int a = 0;
for (Map.Entry<String , Violation> entry : violations.entrySet()) {
String projectName = entry.getKey();
String repoName = "";
@@ -57,7 +57,10 @@ public class ViolationParser {
break;
}
}
if ("".equals(repoName)) continue;
if ("".equals(repoName)) {
a ++;
continue;
}
Violation violation = entry.getValue();
List<Alarm> alarms = violation.getAlarms();
@@ -100,6 +103,7 @@ public class ViolationParser {
gitRepo.close();
}
}
System.out.println(a);
}
private String readPosition(Map<Integer, Integer> positions) {