diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/HunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/HunkParser.java deleted file mode 100644 index 72a16e5..0000000 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/HunkParser.java +++ /dev/null @@ -1,282 +0,0 @@ -package edu.lu.uni.serval.FixPatternParser; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; - -import com.github.gumtreediff.actions.model.Action; - -import edu.lu.uni.serval.config.Configuration; -import edu.lu.uni.serval.diffentry.DiffEntryHunk; -import edu.lu.uni.serval.diffentry.DiffEntryReader; -import edu.lu.uni.serval.gumtree.GumTreeComparer; -import edu.lu.uni.serval.gumtree.regroup.ActionFilter; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; -import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter; -import edu.lu.uni.serval.gumtree.regroup.HunkFixPattern; -import edu.lu.uni.serval.gumtree.regroup.SimpleTree; -import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; - -/** - * Parse fix patterns with GumTree. - * - * @author kui.liu - * - */ -public class HunkParser { - - private String astEditScripts = ""; // it will be used for fix patterns mining. - private String patchesSourceCode = ""; // testing - private String buggyTrees = ""; // Compute similarity for bug localization. - private String sizes = ""; // fix patterns' selection before mining. - private String tokensOfSourceCode = ""; // Compute similarity for bug localization. - private String originalTree = ""; // Guide of generating patches. - private String actionSets = ""; // Guide of generating patches. - - public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { - - // GumTree results - List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); - - if (gumTreeResults != null && gumTreeResults.size() > 0) { - List actionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults); - - ActionFilter filter = new ActionFilter(); - // Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part. - // TODO: variable effects range, sub-actions are these kinds of modification? - List allActionSets = filter.filterOutUselessActions(actionSets); - if (allActionSets.size() == 0) return; - // DiffEntry size: filter out big hunks. - List diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile); - //Filter out the modify actions, which are not in the DiffEntry hunks. - HunkActionFilter hunkFilter = new HunkActionFilter(); - List allHunkFixPatternss = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, allActionSets, revFile, prevFile); - - for (HunkFixPattern hunkFixPattern : allHunkFixPatternss) { - // Range of buggy source code - int startLine = 0; - int endLine = 0; - // Range of fixing source code - int startLine2 = 0; - int endLine2 = 0; - /* - * Convert the ITree of buggy code to a simple tree. - * It will be used to compute the similarity. - */ - List hunkActionSets = hunkFixPattern.getHunkActionSets(); - SimpleTree simpleTree = new SimpleTree(); - simpleTree.setLabel("Block"); - simpleTree.setNodeType("Block"); - List children = new ArrayList<>(); - String astEditScripts = ""; - for (HierarchicalActionSet hunkActionSet : hunkActionSets) { - SimplifyTree abstractIdentifier = new SimplifyTree(); - abstractIdentifier.abstractTree(hunkActionSet); - SimpleTree simpleT = hunkActionSet.getSimpleTree(); - if (simpleT == null) { // Failed to get the simple tree for INS actions. - continue; - } - children.add(simpleT); - - /** - * Select edit scripts for deep learning. - * Edit scripts will be used to mine common fix patterns. - */ - // 1. First level: AST node type. - astEditScripts += getASTEditScripts(hunkActionSet); - // 2. source code: raw tokens - // 3. abstract identifiers: - // 4. semi-source code: - - if (startLine == 0) { - startLine = hunkActionSet.getBugStartLineNum(); - endLine = hunkActionSet.getBugEndLineNum(); - startLine2 = hunkActionSet.getFixStartLineNum(); - endLine2 = hunkActionSet.getFixEndLineNum(); - } else { - if (startLine > hunkActionSet.getBugStartLineNum()) startLine = hunkActionSet.getBugStartLineNum(); - if (startLine2 > hunkActionSet.getFixStartLineNum()) startLine2 = hunkActionSet.getFixStartLineNum(); - if (endLine < hunkActionSet.getBugEndLineNum()) endLine = hunkActionSet.getBugEndLineNum(); - if (endLine2 < hunkActionSet.getFixEndLineNum()) endLine2 = hunkActionSet.getFixEndLineNum(); - } - } - if (children.size() == 0) continue; - if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue; - - simpleTree.setChildren(children); - simpleTree.setParent(null); - - // Source Code of patches. - String patchSourceCode = getPatchSourceCode(hunkFixPattern.getHunk(), startLine, endLine, startLine2, endLine2); - if ("".equals(patchSourceCode)) continue; - - this.patchesSourceCode += "PATCH###\n" + patchSourceCode + "\n"; - int size = astEditScripts.split(" ").length; - this.sizes += size + "\n"; - this.astEditScripts += astEditScripts + "\n"; - -// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n"; - this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n"; -// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n"; -// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n"; - - } - } - } - - private String readActionSet(HierarchicalActionSet actionSet, String line) { - String str = line + actionSet.getActionString() + "\n"; - List subActions = actionSet.getSubActions(); - for (HierarchicalActionSet subAction : subActions) { - str += readActionSet(subAction, line + "---"); - } - return str; - } - - private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) { - // TODO Auto-generated method stub - return null; - } - - private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) { - // TODO Auto-generated method stub - return null; - } - - private String getRawTokenEditScripts(HierarchicalActionSet actionSet) { - // TODO Auto-generated method stub - return null; - } - - private String getPatchSourceCode(DiffEntryHunk hunk, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) { - String sourceCode = hunk.getHunk(); - int bugStartLine = hunk.getBugLineStartNum(); - int fixStartLine = hunk.getFixLineStartNum();String buggyStatements = ""; - String fixedStatements = ""; - BufferedReader reader = null; - try { - reader = new BufferedReader(new StringReader(sourceCode)); - String line = null; - int bugLines = 0; - int fixLines = 0; - int contextLines = 0; // counter of non-buggy code line. - while ((line = reader.readLine()) != null) { - int bugLineIndex = bugLines + contextLines; - int fixLineIndex = fixLines + contextLines; - if (line.startsWith("-")) { - if (bugStartLine + bugLineIndex >= startLineNum && bugStartLine + bugLineIndex <= endLineNum) { - buggyStatements += line + "\n"; - } - bugLines ++; - } else if (line.startsWith("+")) { - if (fixStartLine + fixLineIndex >= startLineNum2 && fixStartLine + fixLineIndex <= endLineNum2) { - fixedStatements += line + "\n"; - } - fixLines ++; - } else { - contextLines ++; - } - - if (bugStartLine + bugLineIndex > endLineNum && fixStartLine + fixLineIndex > endLineNum2) { - break; - } - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - if (reader != null) { - reader.close(); - reader = null; - } - } catch (IOException e) { - e.printStackTrace(); - } - } - return buggyStatements + fixedStatements; - } - - /** - * Get the AST node based edit script of patches in terms of breadth first. - * - * @param actionSet - * @return - */ - private String getASTEditScripts(HierarchicalActionSet actionSet) { - String editScript = ""; - - List actionSets = new ArrayList<>(); - actionSets.add(actionSet); - while (actionSets.size() != 0) { - List subSets = new ArrayList<>(); - for (HierarchicalActionSet set : actionSets) { - subSets.addAll(set.getSubActions()); - String actionStr = set.getActionString(); - int index = actionStr.indexOf("@@"); - String singleEdit = actionStr.substring(0, index).replace(" ", ""); - - if (singleEdit.endsWith("SimpleName")) { - actionStr = actionStr.substring(index + 2); - if (actionStr.startsWith("MethodName")) { - singleEdit = singleEdit.replace("SimpleName", "MethodName"); - } else { - if (actionStr.startsWith("Name")) { - actionStr = actionStr.substring(5, 6); - if (!actionStr.equals(actionStr.toLowerCase())) { - singleEdit = singleEdit.replace("SimpleName", "Name"); - } else { - singleEdit = singleEdit.replace("SimpleName", "Variable"); - } - } else { - singleEdit = singleEdit.replace("SimpleName", "Variable"); - } - } - } - - editScript += singleEdit + " "; - } - actionSets.clear(); - actionSets.addAll(subSets); - } - return editScript; - } - - private void clearITree(HierarchicalActionSet actionSet) { - actionSet.getAction().setNode(null); - for (HierarchicalActionSet subActionSet : actionSet.getSubActions()) { - clearITree(subActionSet); - } - } - - public String getAstEditScripts() { - return astEditScripts; - } - - public String getPatchesSourceCode() { - return patchesSourceCode; - } - - public String getBuggyTrees() { - return buggyTrees; - } - - public String getSizes() { - return sizes; - } - - public String getTokensOfSourceCode() { - return tokensOfSourceCode; - } - - public String getOriginalTree() { - return originalTree; - } - - public String getActionSets() { - return actionSets; - } -} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java index 34dad85..70f59f4 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java @@ -1,26 +1,17 @@ package edu.lu.uni.serval.FixPatternParser; -import java.io.BufferedReader; import java.io.File; -import java.io.IOException; -import java.io.StringReader; import java.util.ArrayList; import java.util.List; import com.github.gumtreediff.actions.model.Action; -import com.github.gumtreediff.actions.model.Update; +import com.github.gumtreediff.actions.model.Move; import com.github.gumtreediff.tree.ITree; -import edu.lu.uni.serval.config.Configuration; -import edu.lu.uni.serval.diffentry.DiffEntryHunk; -import edu.lu.uni.serval.diffentry.DiffEntryReader; import edu.lu.uni.serval.gumtree.GumTreeComparer; import edu.lu.uni.serval.gumtree.regroup.ActionFilter; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; -import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter; -import edu.lu.uni.serval.gumtree.regroup.SimpleTree; -import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; /** * Parse fix patterns with GumTree. @@ -28,156 +19,78 @@ import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; * @author kui.liu * */ -public class Parser { +public abstract class Parser implements ParserInterface { - private String astEditScripts = ""; // it will be used for fix patterns mining. - private String patchesSourceCode = ""; // testing - private String buggyTrees = ""; // Compute similarity for bug localization. - private String sizes = ""; // fix patterns' selection before mining. - private String tokensOfSourceCode = ""; // Compute similarity for bug localization. - private String originalTree = ""; // Guide of generating patches. - private String actionSets = ""; // Guide of generating patches. + protected String astEditScripts = ""; // it will be used for fix patterns mining. + protected String patchesSourceCode = ""; // testing + protected String buggyTrees = ""; // Compute similarity for bug localization. + protected String sizes = ""; // fix patterns' selection before mining. + protected String tokensOfSourceCode = ""; // Compute similarity for bug localization. + protected String originalTree = ""; // Guide of generating patches. + protected String actionSets = ""; // Guide of generating patches. - public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { - + public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile); + + protected List parseChangedSourceCodeWithGumTree(File prevFile, File revFile) { + List actionSets = new ArrayList<>(); // GumTree results List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); - if (gumTreeResults != null && gumTreeResults.size() > 0) { - List actionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults); + // Regroup GumTre results. + List allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults); - /** - * TODO What we need to discuss: - * 3. actions' nodes have the same parent belongs to one fix pattern? - * actions in the same method body. - * field, one by one, - * contains a body block. - */ - ActionFilter filter = new ActionFilter(); // Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part. - List hierarchicalActionSets = filter.filterOutUselessActions(actionSets); // TODO: variable effects range, sub-actions are these kinds of modification? - - // DiffEntry size: - List diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile); // filter out big hunks. - //Filter out the modify actions, which are not in the DiffEntry hunks. - HunkActionFilter hunkFilter = new HunkActionFilter(); - hierarchicalActionSets = hunkFilter.filterActionsByDiffEntryHunk(diffentryHunks, actionSets, revFile, prevFile); - - /** - * Patch size； - * 1. one hunk is one patch。 - * 2. one statement. - */ - - // - - for (HierarchicalActionSet actionSet : hierarchicalActionSets) { - // position of buggy statements - int startPosition = 0; - int endPosition = 0; - // position of fixed statements - int startPosition2 = 0; - int endPosition2 = 0; - - String actionStr = actionSet.getActionString(); - String astNodeType = actionSet.getAstNodeType(); - if (actionStr.startsWith("INS")) { - continue; -// startPosition2 = actionSet.getStartPosition(); -// endPosition2 = startPosition2 + actionSet.getLength(); -// -// if ("EnhancedForStatement".equals(astNodeType) || "ForStatement".equals(astNodeType) -// || "DoStatement".equals(astNodeType) || "WhileStatement".equals(astNodeType) -// || "LabeledStatement".equals(astNodeType) || "SynchronizedStatement".equals(astNodeType) -// || "IfStatement".equals(astNodeType) || "TryStatement".equals(astNodeType)) { -// List firstAndLastMov = getFirstAndLastMoveAction(actionSet); -// if (firstAndLastMov != null) { -// startPosition = firstAndLastMov.get(0).getNode().getPos(); -// ITree lastTree = firstAndLastMov.get(1).getNode(); -// endPosition = lastTree.getPos() + lastTree.getLength(); -// } else { // Pure insert actions without any move actions. -// continue; -// } -// } else { // other insert statements -// continue; -// } - } else if (actionStr.startsWith("UPD")) { - startPosition = actionSet.getStartPosition(); - endPosition = startPosition + actionSet.getLength(); - Update update = (Update) actionSet.getAction(); - ITree newNode = update.getNewNode(); - startPosition2 = newNode.getPos(); - endPosition2 = startPosition2 + newNode.getLength(); - - if ("EnhancedForStatement".equals(astNodeType) || "ForStatement".equals(astNodeType) - || "DoStatement".equals(astNodeType) || "WhileStatement".equals(astNodeType) - || "LabeledStatement".equals(astNodeType) || "SynchronizedStatement".equals(astNodeType) - || "IfStatement".equals(astNodeType) || "TryStatement".equals(astNodeType)) { - List children = update.getNode().getChildren(); - endPosition = getEndPosition(children); - List newChildren = newNode.getChildren(); - endPosition2 = getEndPosition(newChildren); - - if (endPosition == 0 || endPosition2 == 0) { - continue; - } - } - } else {// DEL actions and MOV actions: we don't need these actions, as for now. - continue; - } - - // Get the buggy code and fixed code - if (startPosition != 0 && startPosition2 != 0) { - /* - * Convert the ITree of buggy code to a simple tree. - * It will be used to compute the similarity. - */ - SimplifyTree abstractIdentifier = new SimplifyTree(); - abstractIdentifier.abstractTree(actionSet); - SimpleTree simpleTree = actionSet.getSimpleTree(); - if (simpleTree == null) { // Failed to get the simple tree for INS actions. - continue; - } - - /** - * Select edit scripts for deep learning. - * Edit scripts will be used to mine common fix patterns. - */ - // 1. First level: AST node type. - String astEditScripts = getASTEditScripts(actionSet); - int size = astEditScripts.split(" ").length; - if (size == 1) { - System.out.println(actionSet); - System.out.println(revFile.getPath()); -// continue; - } - this.sizes += size + "\n"; - this.astEditScripts += astEditScripts + "\n"; - // 2. source code: raw tokens - String rawTokenEditScripts = getRawTokenEditScripts(actionSet); - // 3. abstract identifiers: - String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet); - // 4. semi-source code: - String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet); - - - this.buggyTrees += Configuration.BUGGY_TREE_SIGNAL + "\n" + simpleTree.toString() + "\n"; - this.tokensOfSourceCode += getTokensDeepFirst(simpleTree).trim() + "\n"; - this.actionSets += Configuration.BUGGY_TREE_SIGNAL + "\n" + readActionSet(actionSet, "") + "\n"; - this.originalTree += Configuration.BUGGY_TREE_SIGNAL + "\n" + actionSet.getOriginalTree().toString() + "\n"; - -// // Source Code of patches. -// String patchSourceCode = getPatchSourceCode(sourceCode, startLineNum, endLineNum, startLineNum2, -// endLineNum2); -// if (patchSourceCode == null) continue; -// patchesSourceCode += "PATCH###\n" + patchSourceCode; -// patchesSourceCode += actionSet.toString() + "\n"; - } - } + // TODO: variable effects range, sub-actions are these kinds of modification? + actionSets = new ActionFilter().filterOutUselessActions(allActionSets); } + + return actionSets; } - private String readActionSet(HierarchicalActionSet actionSet, String line) { + protected List getFirstAndLastMoveAction(HierarchicalActionSet gumTreeResult) { + List firstAndLastMoveActions = new ArrayList<>(); + List actions = gumTreeResult.getSubActions(); + if (actions.size() == 0) { + return null; + } + Move firstMoveAction = null; + Move lastMoveAction = null; + while (actions.size() > 0) { + List subActions = new ArrayList<>(); + for (HierarchicalActionSet action : actions) { + subActions.addAll(action.getSubActions()); + if (action.toString().startsWith("MOV")) { + if (firstMoveAction == null) { + firstMoveAction = (Move) action.getAction(); + lastMoveAction = (Move) action.getAction(); + } else { + int startPosition = action.getStartPosition(); + int length = action.getLength(); + int startPositionFirst = firstMoveAction.getPosition(); + int startPositionLast = lastMoveAction.getPosition(); + int lengthLast = lastMoveAction.getNode().getLength(); + if (startPosition < startPositionFirst || (startPosition == startPositionFirst && length > firstMoveAction.getLength())) { + firstMoveAction = (Move) action.getAction(); + } + if ((startPosition + length) > (startPositionLast + lengthLast)) { + lastMoveAction = (Move) action.getAction(); + } + } + } + } + + actions.clear(); + actions.addAll(subActions); + } + if (firstMoveAction == null) { + return null; + } + firstAndLastMoveActions.add(firstMoveAction); + firstAndLastMoveActions.add(lastMoveAction); + return firstAndLastMoveActions; + } + + protected String readActionSet(HierarchicalActionSet actionSet, String line) { String str = line + actionSet.getActionString() + "\n"; List subActions = actionSet.getSubActions(); for (HierarchicalActionSet subAction : subActions) { @@ -186,58 +99,22 @@ public class Parser { return str; } - private String getTokensDeepFirst(SimpleTree simpleTree) { - String tokens = ""; - List children = simpleTree.getChildren(); - String astNodeType = simpleTree.getNodeType(); - if ("AssertStatement".equals(astNodeType) || "DoStatement".equals(astNodeType) - || "ForStatement".equals(astNodeType) || "IfStatement".equals(astNodeType) - || "ReturnStatement".equals(astNodeType) || "SwitchStatement".equals(astNodeType) - || "SynchronizedStatement".equals(astNodeType) || "ThrowStatement".equals(astNodeType) - || "TryStatement".equals(astNodeType) || "WhileStatement".equals(astNodeType)) { - String label = simpleTree.getLabel(); - label = label.substring(0, label.indexOf("S")).toLowerCase(); - tokens += label + " "; - } else if ("EnhancedForStatement".equals(astNodeType)) { - tokens += "for "; - } else if ("CatchClause".equals(astNodeType)) { - tokens += "catch "; - } else if ("SwitchCase".equals(astNodeType)) { - tokens += "case "; - } else if ("SuperConstructorInvocation".equals(astNodeType)) { - tokens += "super "; - } else if ("ConstructorInvocation".equals(astNodeType)) { - tokens += "this "; - } else if ("FinallyBody".equals(astNodeType)) { - tokens += "finally "; - } - - if (children.isEmpty()) { - tokens += simpleTree.getNodeType() + " " + simpleTree.getLabel() + " "; - } else { - for (SimpleTree child : children) { - tokens += getTokensDeepFirst(child); - } - } - return tokens; - } - - private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) { + protected String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) { // TODO Auto-generated method stub return null; } - private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) { + protected String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) { // TODO Auto-generated method stub return null; } - private String getRawTokenEditScripts(HierarchicalActionSet actionSet) { + protected String getRawTokenEditScripts(HierarchicalActionSet actionSet) { // TODO Auto-generated method stub return null; } - private int getEndPosition(List children) { + protected int getEndPosition(List children) { int endPosition = 0; for (ITree child : children) { if (child.getLabel().endsWith("Body")) { @@ -248,90 +125,13 @@ public class Parser { return endPosition; } - private String getPatchSourceCode(String sourceCode, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) { - String buggyStatements = ""; - String fixedStatements = ""; - BufferedReader reader = null; - try { - reader = new BufferedReader(new StringReader(sourceCode)); - String line = null; - int startLine = 0; - int counter = 0; - int range = 0; - int startLine2 = 0; - int counter2 = 0; - int range2 = 0; - int counter3 = 0; // counter of non-buggy code line. - while ((line = reader.readLine()) != null) { - if (startLine == 0 && line.startsWith("@@ -")) { - // RegExp.filterSignal(line) - int plusIndex = line.indexOf("+"); - String lineNum = line.substring(4, plusIndex); - String[] nums = lineNum.split(","); - if (nums.length != 2) { - continue; - } - startLine = Integer.parseInt(nums[0].trim()); - range = Integer.parseInt(nums[1].trim()); - if (startLine > endLineNum) { - return null; // Wrong Matching. - } - if (startLine + range < startLineNum) { - startLine = 0; - continue; - } - String lineNum2 = line.substring(plusIndex) .trim(); - lineNum2 = lineNum2.substring(1, lineNum2.length() - 2); - String[] nums2 = lineNum2.split(","); - if (nums2.length != 2) { - startLine = 0; - range = 0; - continue; - } - startLine2 = Integer.parseInt(nums2[0].trim()); - range2 = Integer.parseInt(nums2[1].trim()); - continue; - } - - int lineNum1 = counter + counter3; - int lineNum2 = counter2 + counter3; - if (startLine > 0 && startLine2 > 0 && lineNum1 < range && lineNum2 < range2) { - if (line.startsWith("-") && startLine + lineNum1 >= startLineNum && startLine + lineNum1 <= endLineNum) { - buggyStatements += line + "\n"; - } else if (line.startsWith("+") && startLine2 + lineNum2 >= startLineNum2 && startLine2 + lineNum2 <= endLineNum2) { - fixedStatements += line + "\n"; - } - if (line.startsWith("-")) { - counter ++; - } else if (line.startsWith("+")) { - counter2 ++; - } else { - counter3 ++; - } - } - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - if (reader != null) { - reader.close(); - reader = null; - } - } catch (IOException e) { - e.printStackTrace(); - } - } - return buggyStatements + fixedStatements; - } - /** * Get the AST node based edit script of patches in terms of breadth first. * * @param actionSet * @return */ - private String getASTEditScripts(HierarchicalActionSet actionSet) { + protected String getASTEditScripts(HierarchicalActionSet actionSet) { String editScript = ""; List actionSets = new ArrayList<>(); @@ -370,37 +170,44 @@ public class Parser { return editScript; } - private void clearITree(HierarchicalActionSet actionSet) { + protected void clearITree(HierarchicalActionSet actionSet) { actionSet.getAction().setNode(null); for (HierarchicalActionSet subActionSet : actionSet.getSubActions()) { clearITree(subActionSet); } } - + + @Override public String getAstEditScripts() { return astEditScripts; } + @Override public String getPatchesSourceCode() { return patchesSourceCode; } + @Override public String getBuggyTrees() { return buggyTrees; } + @Override public String getSizes() { return sizes; } + @Override public String getTokensOfSourceCode() { return tokensOfSourceCode; } + @Override public String getOriginalTree() { return originalTree; } + @Override public String getActionSets() { return actionSets; } diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java new file mode 100644 index 0000000..2245087 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java @@ -0,0 +1,23 @@ +package edu.lu.uni.serval.FixPatternParser; + +import java.io.File; + +public interface ParserInterface { + + + public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile); + + public String getAstEditScripts(); + + public String getPatchesSourceCode(); + + public String getBuggyTrees(); + + public String getSizes(); + + public String getTokensOfSourceCode(); + + public String getOriginalTree(); + + public String getActionSets(); +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java index 259193f..82f66fc 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java @@ -7,9 +7,9 @@ public class RunnableParser implements Runnable { private File prevFile; private File revFile; private File diffentryFile; - SingleStatementParser parser; + private Parser parser; - public RunnableParser(File prevFile, File revFile, File diffentryFile, SingleStatementParser parser) { + public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) { this.prevFile = prevFile; this.revFile = revFile; this.diffentryFile = diffentryFile; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/SingleStatementParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/SingleStatementParser.java deleted file mode 100644 index 0b4d202..0000000 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/SingleStatementParser.java +++ /dev/null @@ -1,401 +0,0 @@ -package edu.lu.uni.serval.FixPatternParser; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; - -import org.eclipse.jdt.core.dom.CompilationUnit; - -import com.github.gumtreediff.actions.model.Action; -import com.github.gumtreediff.actions.model.Move; -import com.github.gumtreediff.actions.model.Update; -import com.github.gumtreediff.tree.ITree; - -import edu.lu.uni.serval.FixPattern.utils.Checker; -import edu.lu.uni.serval.config.Configuration; -import edu.lu.uni.serval.diffentry.DiffEntryHunk; -import edu.lu.uni.serval.diffentry.DiffEntryReader; -import edu.lu.uni.serval.gumtree.GumTreeComparer; -import edu.lu.uni.serval.gumtree.regroup.ActionFilter; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; -import edu.lu.uni.serval.gumtree.regroup.SimpleTree; -import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; - -/** - * Parse fix patterns with GumTree. - * - * @author kui.liu - * - */ -public class SingleStatementParser { - - private String astEditScripts = ""; // it will be used for fix patterns mining. - private String patchesSourceCode = ""; // testing - private String buggyTrees = ""; // Compute similarity for bug localization. - private String sizes = ""; // fix patterns' selection before mining. - private String tokensOfSourceCode = ""; // Compute similarity for bug localization. - private String originalTree = ""; // Guide of generating patches. - private String actionSets = ""; // Guide of generating patches. - - public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { - // GumTree results - List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); - - if (gumTreeResults != null && gumTreeResults.size() > 0) { - List allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults); - // Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part. - // TODO: variable effects range, sub-actions are these kinds of modification? - List actionSets = new ActionFilter().filterOutUselessActions(allActionSets); - - if (actionSets.size() > 0) { - // DiffEntry Hunks: filter out big hunks. - List diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile); - for (HierarchicalActionSet actionSet : actionSets) { - // position of buggy statements - int startPosition = 0; - int endPosition = 0; - // position of fixed statements - int startPosition2 = 0; - int endPosition2 = 0; - - String actionStr = actionSet.getActionString(); - String astNodeType = actionSet.getAstNodeType(); - if (actionStr.startsWith("INS")) { - startPosition2 = actionSet.getStartPosition(); - endPosition2 = startPosition2 + actionSet.getLength(); - List firstAndLastMov = getFirstAndLastMoveAction(actionSet); - if (firstAndLastMov != null) { - startPosition = firstAndLastMov.get(0).getNode().getPos(); - ITree lastTree = firstAndLastMov.get(1).getNode(); - endPosition = lastTree.getPos() + lastTree.getLength(); - } else { // Ignore the pure insert actions without any move actions. - continue; - } - } else if (actionStr.startsWith("UPD")) { - startPosition = actionSet.getStartPosition(); - endPosition = startPosition + actionSet.getLength(); - Update update = (Update) actionSet.getAction(); - ITree newNode = update.getNewNode(); - startPosition2 = newNode.getPos(); - endPosition2 = startPosition2 + newNode.getLength(); - - if (Checker.containsBodyBlock(astNodeType)) { - List children = update.getNode().getChildren(); - endPosition = getEndPosition(children); - List newChildren = newNode.getChildren(); - endPosition2 = getEndPosition(newChildren); - - if (endPosition == 0) { - endPosition = startPosition + actionSet.getLength(); - } - if (endPosition2 == 0) { - endPosition2 = startPosition2 + newNode.getLength(); - } - } - } else {// DEL actions and MOV actions: we don't need these actions, as for now. - continue; - } - if (startPosition == 0 || startPosition2 == 0) { - continue; - } - - CUCreator cuCreator = new CUCreator(); - CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); - CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); - if (prevUnit == null || revUnit == null) { - continue; - } - - // Get line numbers. - int startLine = prevUnit.getLineNumber(startPosition); - int endLine = prevUnit.getLineNumber(endPosition); - int startLine2 = revUnit.getLineNumber(startPosition2); - int endLine2 = revUnit.getLineNumber(endPosition2); - if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue; - //Filter out the modify actions, which are not in the DiffEntry hunks. - DiffEntryHunk hunk = matchHunk(startLine, endLine, startLine2, endLine2, actionStr, diffentryHunks); - if (hunk == null) { - continue; - } - - /* - * Convert the ITree of buggy code to a simple tree. - * It will be used to compute the similarity. - */ - SimplifyTree abstractIdentifier = new SimplifyTree(); - abstractIdentifier.abstractTree(actionSet); - SimpleTree simpleTree = actionSet.getSimpleTree(); - if (simpleTree == null) { // Failed to get the simple tree for INS actions. - continue; - } - - /** - * Select edit scripts for deep learning. - * Edit scripts will be used to mine common fix patterns. - */ - // 1. First level: AST node type. - String astEditScripts = getASTEditScripts(actionSet); - int size = astEditScripts.split(" ").length; - if (size == 1) { -// System.out.println(actionSet); - continue; - } - - // Source Code of patches. - String patchSourceCode = getPatchSourceCode(hunk, startLine, endLine, startLine2, endLine2); - if ("".equals(patchSourceCode)) continue; - - this.patchesSourceCode += Configuration.PATCH_SIGNAL + "\n" + patchSourceCode + "\n"; - this.sizes += size + "\n"; - this.astEditScripts += astEditScripts + "\n"; - // 2. source code: raw tokens - String rawTokenEditScripts = getRawTokenEditScripts(actionSet); - // 3. abstract identifiers: - String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet); - // 4. semi-source code: - String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet); - -// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n"; - this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n"; -// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n"; -// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n"; - } - actionSets.clear(); - } - allActionSets.clear(); - gumTreeResults.clear(); - } - } - - private DiffEntryHunk matchHunk(int startLine, int endLine, int startLine2, int endLine2, String actionStr, List hunks) { - for (DiffEntryHunk hunk : hunks) { - int bugStartLine = hunk.getBugLineStartNum(); - int bugRange = hunk.getBugRange(); - int fixStartLine = hunk.getFixLineStartNum(); - int fixRange = hunk.getFixRange(); - - if (actionStr.startsWith("INS")) { - if (fixStartLine + fixRange < startLine2) { - continue; - } - if (endLine2 < fixStartLine ) { - return null; - } - return hunk; - } else { - if (bugStartLine + bugRange < startLine) { - continue; - } - if (endLine < bugStartLine ) { - return null; - } - return hunk; - } - } - return null; - } - - private List getFirstAndLastMoveAction(HierarchicalActionSet gumTreeResult) { - List firstAndLastMoveActions = new ArrayList<>(); - List actions = gumTreeResult.getSubActions(); - if (actions.size() == 0) { - return null; - } - Move firstMoveAction = null; - Move lastMoveAction = null; - while (actions.size() > 0) { - List subActions = new ArrayList<>(); - for (HierarchicalActionSet action : actions) { - subActions.addAll(action.getSubActions()); - if (action.toString().startsWith("MOV")) { - if (firstMoveAction == null) { - firstMoveAction = (Move) action.getAction(); - lastMoveAction = (Move) action.getAction(); - } else { - int startPosition = action.getStartPosition(); - int length = action.getLength(); - int startPositionFirst = firstMoveAction.getPosition(); - int startPositionLast = lastMoveAction.getPosition(); - int lengthLast = lastMoveAction.getNode().getLength(); - if (startPosition < startPositionFirst || (startPosition == startPositionFirst && length > firstMoveAction.getLength())) { - firstMoveAction = (Move) action.getAction(); - } - if ((startPosition + length) > (startPositionLast + lengthLast)) { - lastMoveAction = (Move) action.getAction(); - } - } - } - } - - actions.clear(); - actions.addAll(subActions); - } - if (firstMoveAction == null) { - return null; - } - firstAndLastMoveActions.add(firstMoveAction); - firstAndLastMoveActions.add(lastMoveAction); - return firstAndLastMoveActions; - } - - private String readActionSet(HierarchicalActionSet actionSet, String line) { - String str = line + actionSet.getActionString() + "\n"; - List subActions = actionSet.getSubActions(); - for (HierarchicalActionSet subAction : subActions) { - str += readActionSet(subAction, line + "---"); - } - return str; - } - - private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) { - // TODO Auto-generated method stub - return null; - } - - private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) { - // TODO Auto-generated method stub - return null; - } - - private String getRawTokenEditScripts(HierarchicalActionSet actionSet) { - // TODO Auto-generated method stub - return null; - } - - private int getEndPosition(List children) { - int endPosition = 0; - for (ITree child : children) { - if (child.getLabel().endsWith("Body")) { - endPosition = child.getPos() - 1; - break; - } - } - return endPosition; - } - - private String getPatchSourceCode(DiffEntryHunk hunk, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) { - String sourceCode = hunk.getHunk(); - int bugStartLine = hunk.getBugLineStartNum(); - int fixStartLine = hunk.getFixLineStartNum(); - String buggyStatements = ""; - String fixedStatements = ""; - BufferedReader reader = null; - try { - reader = new BufferedReader(new StringReader(sourceCode)); - String line = null; - int bugLines = 0; - int fixLines = 0; - int contextLines = 0; // counter of non-buggy code line. - while ((line = reader.readLine()) != null) { - int bugLineIndex = bugLines + contextLines; - int fixLineIndex = fixLines + contextLines; - if (line.startsWith("-")) { - if (bugStartLine + bugLineIndex >= startLineNum && bugStartLine + bugLineIndex <= endLineNum) { - buggyStatements += line + "\n"; - } - bugLines ++; - } else if (line.startsWith("+")) { - if (fixStartLine + fixLineIndex >= startLineNum2 && fixStartLine + fixLineIndex <= endLineNum2) { - fixedStatements += line + "\n"; - } - fixLines ++; - } else { - contextLines ++; - } - - if (bugStartLine + bugLineIndex > endLineNum && fixStartLine + fixLineIndex > endLineNum2) { - break; - } - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - if (reader != null) { - reader.close(); - reader = null; - } - } catch (IOException e) { - e.printStackTrace(); - } - } - return buggyStatements + fixedStatements; - } - - /** - * Get the AST node based edit script of patches in terms of breadth first. - * - * @param actionSet - * @return - */ - private String getASTEditScripts(HierarchicalActionSet actionSet) { - String editScript = ""; - - List actionSets = new ArrayList<>(); - actionSets.add(actionSet); - while (actionSets.size() != 0) { - List subSets = new ArrayList<>(); - for (HierarchicalActionSet set : actionSets) { - subSets.addAll(set.getSubActions()); - String actionStr = set.getActionString(); - int index = actionStr.indexOf("@@"); - String singleEdit = actionStr.substring(0, index).replace(" ", ""); - - if (singleEdit.endsWith("SimpleName")) { - actionStr = actionStr.substring(index + 2); - if (actionStr.startsWith("MethodName")) { - singleEdit = singleEdit.replace("SimpleName", "MethodName"); - } else { - if (actionStr.startsWith("Name")) { - actionStr = actionStr.substring(5, 6); - if (!actionStr.equals(actionStr.toLowerCase())) { - singleEdit = singleEdit.replace("SimpleName", "Name"); - } else { - singleEdit = singleEdit.replace("SimpleName", "Variable"); - } - } else { - singleEdit = singleEdit.replace("SimpleName", "Variable"); - } - } - } - - editScript += singleEdit + " "; - } - actionSets.clear(); - actionSets.addAll(subSets); - } - return editScript; - } - - public String getAstEditScripts() { - return astEditScripts; - } - - public String getPatchesSourceCode() { - return patchesSourceCode; - } - - public String getBuggyTrees() { - return buggyTrees; - } - - public String getSizes() { - return sizes; - } - - public String getTokensOfSourceCode() { - return tokensOfSourceCode; - } - - public String getOriginalTree() { - return originalTree; - } - - public String getActionSets() { - return actionSets; - } -} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/TestParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/TestParser.java deleted file mode 100644 index 50a7ccd..0000000 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/TestParser.java +++ /dev/null @@ -1,111 +0,0 @@ -package edu.lu.uni.serval.FixPatternParser; - -import static java.lang.System.err; - -import java.io.File; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import edu.lu.uni.serval.utils.FileHelper; - -/** - * Test fix pattern parser. - */ -public class TestParser { - private static final long MILLIS_TO_WAIT = 60L; - public static void main(String[] args) { - String testDataPath = "../GumTreeInput/"; //DiffEntries prevFiles revFiles - File inputFileDirector = new File(testDataPath); - File[] files = inputFileDirector.listFiles(); // project folders - - String outputPath = "../GumTreeOutput/"; - - StringBuilder astEditScripts = new StringBuilder(); -// StringBuilder originalTrees = new StringBuilder(); -// StringBuilder buggyTrees = new StringBuilder(); -// StringBuilder actionSets = new StringBuilder(); - StringBuilder tokens = new StringBuilder(); - StringBuilder sizes = new StringBuilder(); - StringBuilder patches = new StringBuilder(); - - for (File file : files) { - String projectFolder = file.getPath(); - File revFileFolder = new File(projectFolder + "/revFiles/");// revised file folder - File[] revFiles = revFileFolder.listFiles(); - int a = 0; - System.out.println(file.getPath()); - String outputFile = outputPath + file.getName(); - FileHelper.deleteDirectory(outputFile); - for (File revFile : revFiles) { - String fileName = revFile.getName(); - if (fileName.endsWith(".java")) { - - File prevFile = new File(projectFolder + "/prevFiles/prev_" + fileName);// previous file - File diffentryFile = new File(projectFolder + "/DiffEntries/" + fileName.replace(".java", ".txt")); // DiffEntry file - - SingleStatementParser parser = new SingleStatementParser(); -// HunkParser parser = new HunkParser(); - - final ExecutorService executor = Executors.newSingleThreadExecutor(); - // schedule the work - final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser)); - try { - // where we wait for task to complete - future.get(MILLIS_TO_WAIT, TimeUnit.SECONDS); - astEditScripts.append(parser.getAstEditScripts()); -// originalTrees.append(parser.getOriginalTree()); -// buggyTrees.append(parser.getBuggyTrees()); -// actionSets.append(parser.getActionSets()); - tokens.append(parser.getTokensOfSourceCode()); - sizes.append(parser.getSizes()); - patches.append(parser.getPatchesSourceCode()); - a ++; - System.out.println(a); - if (a % 100 == 0) { - FileHelper.outputToFile(outputFile + "/EditScripts.list", astEditScripts, true); - FileHelper.outputToFile(outputFile + "/Tokens.list", tokens, true); - FileHelper.outputToFile(outputFile + "/Sizes.list", sizes, true); - FileHelper.outputToFile(outputFile + "/Patches.list", patches, true); - astEditScripts.setLength(0); - tokens.setLength(0); - sizes.setLength(0); - patches.setLength(0); - System.out.println(a); - } - } catch (TimeoutException e) { - err.println("task timed out"); - future.cancel(true /* mayInterruptIfRunning */ ); - } catch (InterruptedException e) { - err.println("task interrupted"); - } catch (ExecutionException e) { - err.println("task aborted"); - } finally { - executor.shutdownNow(); - } -// parser.parseFixPatterns(prevFile, revFile, diffentryFile); - - } - } -// break; - FileHelper.outputToFile(outputFile + "/EditScripts.list", astEditScripts, true); -// FileHelper.outputToFile(outputFile + "/OriginalTrees.list", originalTrees, false); -// FileHelper.outputToFile(outputFile + "/BuggyTrees.list", buggyTrees, false); -// FileHelper.outputToFile(outputFile + "/ActionSets.list", actionSets, false); - FileHelper.outputToFile(outputFile + "/Tokens.list", tokens, true); - FileHelper.outputToFile(outputFile + "/Sizes.list", sizes, true); - FileHelper.outputToFile(outputFile + "/Patches.list", patches, true); -// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/EditScripts.list", astEditScripts, false); -// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/OriginalTrees.list", originalTrees, false); -// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/BuggyTrees.list", buggyTrees, false); -// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/ActionSets.list", actionSets, false); -// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/Tokens.list", tokens, false); -// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/Sizes.list", sizes, false); -// FileHelper.outputToFile("OUTPUT/GumTreeResults_Exp2/Patches.list", patches, false); - } - } - -} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchHunkParser.java new file mode 100644 index 0000000..ebe4284 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchHunkParser.java @@ -0,0 +1,113 @@ +package edu.lu.uni.serval.FixPatternParser.patch; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import edu.lu.uni.serval.FixPatternParser.Tokenizer; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.diffentry.DiffEntryHunk; +import edu.lu.uni.serval.diffentry.DiffEntryReader; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter; +import edu.lu.uni.serval.gumtree.regroup.HunkFixPattern; +import edu.lu.uni.serval.gumtree.regroup.SimpleTree; +import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; + +/** + * Parse fix patterns with GumTree. + * + * Multiple statements bugs. + * + * @author kui.liu + * + */ +public class CommitPatchHunkParser extends CommitPatchParser { + + + @Override + public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { + + // GumTree results + List actionSets = parseChangedSourceCodeWithGumTree(prevFile, revFile); + + if (actionSets.size() > 0) { + // DiffEntry size: filter out big hunks. + List diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile); + //Filter out the modify actions, which are not in the DiffEntry hunks. + HunkActionFilter hunkFilter = new HunkActionFilter(); + List allHunkFixPatternss = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, actionSets, revFile, prevFile); + + for (HunkFixPattern hunkFixPattern : allHunkFixPatternss) { + // Range of buggy source code + int startLine = 0; + int endLine = 0; + // Range of fixing source code + int startLine2 = 0; + int endLine2 = 0; + /* + * Convert the ITree of buggy code to a simple tree. + * It will be used to compute the similarity. + */ + List hunkActionSets = hunkFixPattern.getHunkActionSets(); + SimpleTree simpleTree = new SimpleTree(); + simpleTree.setLabel("Block"); + simpleTree.setNodeType("Block"); + List children = new ArrayList<>(); + String astEditScripts = ""; + for (HierarchicalActionSet hunkActionSet : hunkActionSets) { + SimplifyTree abstractIdentifier = new SimplifyTree(); + abstractIdentifier.abstractTree(hunkActionSet); + SimpleTree simpleT = hunkActionSet.getSimpleTree(); + if (simpleT == null) { // Failed to get the simple tree for INS actions. + continue; + } + children.add(simpleT); + + /** + * Select edit scripts for deep learning. + * Edit scripts will be used to mine common fix patterns. + */ + // 1. First level: AST node type. + astEditScripts += getASTEditScripts(hunkActionSet); + // 2. source code: raw tokens + // 3. abstract identifiers: + // 4. semi-source code: + + if (startLine == 0) { + startLine = hunkActionSet.getBugStartLineNum(); + endLine = hunkActionSet.getBugEndLineNum(); + startLine2 = hunkActionSet.getFixStartLineNum(); + endLine2 = hunkActionSet.getFixEndLineNum(); + } else { + if (startLine > hunkActionSet.getBugStartLineNum()) startLine = hunkActionSet.getBugStartLineNum(); + if (startLine2 > hunkActionSet.getFixStartLineNum()) startLine2 = hunkActionSet.getFixStartLineNum(); + if (endLine < hunkActionSet.getBugEndLineNum()) endLine = hunkActionSet.getBugEndLineNum(); + if (endLine2 < hunkActionSet.getFixEndLineNum()) endLine2 = hunkActionSet.getFixEndLineNum(); + } + } + if (children.size() == 0) continue; + if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue; + + simpleTree.setChildren(children); + simpleTree.setParent(null); + + // Source Code of patches. + String patchSourceCode = getPatchSourceCode(hunkFixPattern.getHunk(), startLine, endLine, startLine2, endLine2); + if ("".equals(patchSourceCode)) continue; + + this.patchesSourceCode += "PATCH###\n" + patchSourceCode + "\n"; + int size = astEditScripts.split(" ").length; + this.sizes += size + "\n"; + this.astEditScripts += astEditScripts + "\n"; + +// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n"; + this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n"; +// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n"; +// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n"; + + } + } + } + +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchParser.java new file mode 100644 index 0000000..c5db6b0 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchParser.java @@ -0,0 +1,101 @@ +package edu.lu.uni.serval.FixPatternParser.patch; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.List; + +import edu.lu.uni.serval.FixPatternParser.Parser; +import edu.lu.uni.serval.diffentry.DiffEntryHunk; + +/** + * Parse fix patterns with GumTree. + * + * @author kui.liu + * + */ +public class CommitPatchParser extends Parser{ + + @Override + public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { + } + + protected DiffEntryHunk matchHunk(int startLine, int endLine, int startLine2, int endLine2, String actionStr, List hunks) { + for (DiffEntryHunk hunk : hunks) { + int bugStartLine = hunk.getBugLineStartNum(); + int bugRange = hunk.getBugRange(); + int fixStartLine = hunk.getFixLineStartNum(); + int fixRange = hunk.getFixRange(); + + if (actionStr.startsWith("INS")) { + if (fixStartLine + fixRange < startLine2) { + continue; + } + if (endLine2 < fixStartLine ) { + return null; + } + return hunk; + } else { + if (bugStartLine + bugRange < startLine) { + continue; + } + if (endLine < bugStartLine ) { + return null; + } + return hunk; + } + } + return null; + } + + protected String getPatchSourceCode(DiffEntryHunk hunk, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) { + String sourceCode = hunk.getHunk(); + int bugStartLine = hunk.getBugLineStartNum(); + int fixStartLine = hunk.getFixLineStartNum(); + String buggyStatements = ""; + String fixedStatements = ""; + BufferedReader reader = null; + try { + reader = new BufferedReader(new StringReader(sourceCode)); + String line = null; + int bugLines = 0; + int fixLines = 0; + int contextLines = 0; // counter of non-buggy code line. + while ((line = reader.readLine()) != null) { + int bugLineIndex = bugLines + contextLines; + int fixLineIndex = fixLines + contextLines; + if (line.startsWith("-")) { + if (bugStartLine + bugLineIndex >= startLineNum && bugStartLine + bugLineIndex <= endLineNum) { + buggyStatements += line + "\n"; + } + bugLines ++; + } else if (line.startsWith("+")) { + if (fixStartLine + fixLineIndex >= startLineNum2 && fixStartLine + fixLineIndex <= endLineNum2) { + fixedStatements += line + "\n"; + } + fixLines ++; + } else { + contextLines ++; + } + + if (bugStartLine + bugLineIndex > endLineNum && fixStartLine + fixLineIndex > endLineNum2) { + break; + } + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + if (reader != null) { + reader.close(); + reader = null; + } + } catch (IOException e) { + e.printStackTrace(); + } + } + return buggyStatements + fixedStatements; + } + +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchSingleStatementParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchSingleStatementParser.java new file mode 100644 index 0000000..5af42b4 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchSingleStatementParser.java @@ -0,0 +1,164 @@ +package edu.lu.uni.serval.FixPatternParser.patch; + +import java.io.File; +import java.util.List; + +import org.eclipse.jdt.core.dom.CompilationUnit; + +import com.github.gumtreediff.actions.model.Move; +import com.github.gumtreediff.actions.model.Update; +import com.github.gumtreediff.tree.ITree; + +import edu.lu.uni.serval.FixPattern.utils.Checker; +import edu.lu.uni.serval.FixPatternParser.CUCreator; +import edu.lu.uni.serval.FixPatternParser.Tokenizer; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.diffentry.DiffEntryHunk; +import edu.lu.uni.serval.diffentry.DiffEntryReader; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.gumtree.regroup.SimpleTree; +import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; + +/** + * Parse fix patterns with GumTree. + * + * Single Statement bugs. + * + * @author kui.liu + * + */ +public class CommitPatchSingleStatementParser extends CommitPatchParser { + + @Override + public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { + // GumTree results + List actionSets = parseChangedSourceCodeWithGumTree(prevFile, revFile); + + if (actionSets.size() > 0) { + // DiffEntry Hunks: filter out big hunks. + List diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile); + for (HierarchicalActionSet actionSet : actionSets) { + // position of buggy statements + int startPosition = 0; + int endPosition = 0; + // position of fixed statements + int startPosition2 = 0; + int endPosition2 = 0; + + String actionStr = actionSet.getActionString(); + String astNodeType = actionSet.getAstNodeType(); + if (actionStr.startsWith("INS")) { + startPosition2 = actionSet.getStartPosition(); + endPosition2 = startPosition2 + actionSet.getLength(); + List firstAndLastMov = getFirstAndLastMoveAction(actionSet); + if (firstAndLastMov != null) { + startPosition = firstAndLastMov.get(0).getNode().getPos(); + ITree lastTree = firstAndLastMov.get(1).getNode(); + endPosition = lastTree.getPos() + lastTree.getLength(); + } else { // Ignore the pure insert actions without any move + // actions. + continue; + } + } else if (actionStr.startsWith("UPD")) { + startPosition = actionSet.getStartPosition(); + endPosition = startPosition + actionSet.getLength(); + Update update = (Update) actionSet.getAction(); + ITree newNode = update.getNewNode(); + startPosition2 = newNode.getPos(); + endPosition2 = startPosition2 + newNode.getLength(); + + if (Checker.containsBodyBlock(astNodeType)) { + List children = update.getNode().getChildren(); + endPosition = getEndPosition(children); + List newChildren = newNode.getChildren(); + endPosition2 = getEndPosition(newChildren); + + if (endPosition == 0) { + endPosition = startPosition + actionSet.getLength(); + } + if (endPosition2 == 0) { + endPosition2 = startPosition2 + newNode.getLength(); + } + } + } else {// DEL actions and MOV actions: we don't need these + // actions, as for now. + continue; + } + if (startPosition == 0 || startPosition2 == 0) { + continue; + } + + CUCreator cuCreator = new CUCreator(); + CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); + CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); + if (prevUnit == null || revUnit == null) { + continue; + } + + // Get line numbers. + int startLine = prevUnit.getLineNumber(startPosition); + int endLine = prevUnit.getLineNumber(endPosition); + int startLine2 = revUnit.getLineNumber(startPosition2); + int endLine2 = revUnit.getLineNumber(endPosition2); + if (endLine - startLine >= Configuration.HUNK_SIZE - 2 + || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2) + continue; + // Filter out the modify actions, which are not in the DiffEntry + // hunks. + DiffEntryHunk hunk = matchHunk(startLine, endLine, startLine2, endLine2, actionStr, diffentryHunks); + if (hunk == null) { + continue; + } + + /* + * Convert the ITree of buggy code to a simple tree. It will be + * used to compute the similarity. + */ + SimplifyTree abstractIdentifier = new SimplifyTree(); + abstractIdentifier.abstractTree(actionSet); + SimpleTree simpleTree = actionSet.getSimpleTree(); + if (simpleTree == null) { // Failed to get the simple tree for + // INS actions. + continue; + } + + /** + * Select edit scripts for deep learning. Edit scripts will be + * used to mine common fix patterns. + */ + // 1. First level: AST node type. + String astEditScripts = getASTEditScripts(actionSet); + int size = astEditScripts.split(" ").length; + if (size == 1) { + // System.out.println(actionSet); + continue; + } + + // Source Code of patches. + String patchSourceCode = getPatchSourceCode(hunk, startLine, endLine, startLine2, endLine2); + if ("".equals(patchSourceCode)) + continue; + + this.patchesSourceCode += Configuration.PATCH_SIGNAL + "\n" + patchSourceCode + "\n"; + this.sizes += size + "\n"; + this.astEditScripts += astEditScripts + "\n"; + // 2. source code: raw tokens + String rawTokenEditScripts = getRawTokenEditScripts(actionSet); + // 3. abstract identifiers: + String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet); + // 4. semi-source code: + String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet); + + // this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + + // simpleTree.toString() + "\n"; + this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n"; + // this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + + // readActionSet(actionSet, "") + "\n"; + // this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + + // actionSet.getOriginalTree().toString() + "\n"; + } + actionSets.clear(); + } + } + +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java index d268fe0..4f7ed37 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java @@ -6,21 +6,19 @@ import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.List; - -import com.github.gumtreediff.actions.model.Action; +import java.util.Map; import edu.lu.uni.serval.FixPatternParser.Tokenizer; import edu.lu.uni.serval.config.Configuration; import edu.lu.uni.serval.diffentry.DiffEntryHunk; import edu.lu.uni.serval.diffentry.DiffEntryReader; -import edu.lu.uni.serval.gumtree.GumTreeComparer; import edu.lu.uni.serval.gumtree.regroup.ActionFilter; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter; import edu.lu.uni.serval.gumtree.regroup.HunkFixPattern; import edu.lu.uni.serval.gumtree.regroup.SimpleTree; import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; +import edu.lu.uni.serval.utils.MapSorter; /** * Parse fix patterns with GumTree. @@ -28,34 +26,37 @@ import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; * @author kui.liu * */ -public class FixedViolationHunkParser { +public class FixedViolationHunkParser extends FixedViolationParser { - private String astEditScripts = ""; // it will be used for fix patterns mining. - private String patchesSourceCode = ""; // testing - private String buggyTrees = ""; // Compute similarity for bug localization. - private String sizes = ""; // fix patterns' selection before mining. - private String tokensOfSourceCode = ""; // Compute similarity for bug localization. - private String originalTree = ""; // Guide of generating patches. - private String actionSets = ""; // Guide of generating patches. - - public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { + @Override + public void parseFixPatterns(File prevFile, File revFile, File positionsFile) { // GumTree results - List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); + List actionSets = parseChangedSourceCodeWithGumTree(prevFile, revFile); - if (gumTreeResults != null && gumTreeResults.size() > 0) { - List actionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults); + if (actionSets.size() > 0) { + Map positions = readPositions(positionsFile); + if (positions.size() > 1) { + MapSorter sorter = new MapSorter<>(); + positions = sorter.sortByKeyAscending(positions); + } + + for (Map.Entry entry : positions.entrySet()) { + // only statements + for (HierarchicalActionSet actionSet : actionSets) { + String astNodeType = actionSet.getAstNodeType(); + if (astNodeType.endsWith("Statement") || "FieldDeclaration".equals(astNodeType)) { + + } + } + } ActionFilter filter = new ActionFilter(); - // Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part. - // TODO: variable effects range, sub-actions are these kinds of modification? - List allActionSets = filter.filterOutUselessActions(actionSets); - if (allActionSets.size() == 0) return; // DiffEntry size: filter out big hunks. - List diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile); + List diffentryHunks = new DiffEntryReader().readHunks(positionsFile); //Filter out the modify actions, which are not in the DiffEntry hunks. HunkActionFilter hunkFilter = new HunkActionFilter(); - List allHunkFixPatternss = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, allActionSets, revFile, prevFile); + List allHunkFixPatternss = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, actionSets, revFile, prevFile); for (HunkFixPattern hunkFixPattern : allHunkFixPatternss) { // Range of buggy source code @@ -129,30 +130,6 @@ public class FixedViolationHunkParser { } } - private String readActionSet(HierarchicalActionSet actionSet, String line) { - String str = line + actionSet.getActionString() + "\n"; - List subActions = actionSet.getSubActions(); - for (HierarchicalActionSet subAction : subActions) { - str += readActionSet(subAction, line + "---"); - } - return str; - } - - private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) { - // TODO Auto-generated method stub - return null; - } - - private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) { - // TODO Auto-generated method stub - return null; - } - - private String getRawTokenEditScripts(HierarchicalActionSet actionSet) { - // TODO Auto-generated method stub - return null; - } - private String getPatchSourceCode(DiffEntryHunk hunk, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) { String sourceCode = hunk.getHunk(); int bugStartLine = hunk.getBugLineStartNum(); @@ -201,83 +178,4 @@ public class FixedViolationHunkParser { return buggyStatements + fixedStatements; } - /** - * Get the AST node based edit script of patches in terms of breadth first. - * - * @param actionSet - * @return - */ - private String getASTEditScripts(HierarchicalActionSet actionSet) { - String editScript = ""; - - List actionSets = new ArrayList<>(); - actionSets.add(actionSet); - while (actionSets.size() != 0) { - List subSets = new ArrayList<>(); - for (HierarchicalActionSet set : actionSets) { - subSets.addAll(set.getSubActions()); - String actionStr = set.getActionString(); - int index = actionStr.indexOf("@@"); - String singleEdit = actionStr.substring(0, index).replace(" ", ""); - - if (singleEdit.endsWith("SimpleName")) { - actionStr = actionStr.substring(index + 2); - if (actionStr.startsWith("MethodName")) { - singleEdit = singleEdit.replace("SimpleName", "MethodName"); - } else { - if (actionStr.startsWith("Name")) { - actionStr = actionStr.substring(5, 6); - if (!actionStr.equals(actionStr.toLowerCase())) { - singleEdit = singleEdit.replace("SimpleName", "Name"); - } else { - singleEdit = singleEdit.replace("SimpleName", "Variable"); - } - } else { - singleEdit = singleEdit.replace("SimpleName", "Variable"); - } - } - } - - editScript += singleEdit + " "; - } - actionSets.clear(); - actionSets.addAll(subSets); - } - return editScript; - } - - private void clearITree(HierarchicalActionSet actionSet) { - actionSet.getAction().setNode(null); - for (HierarchicalActionSet subActionSet : actionSet.getSubActions()) { - clearITree(subActionSet); - } - } - - public String getAstEditScripts() { - return astEditScripts; - } - - public String getPatchesSourceCode() { - return patchesSourceCode; - } - - public String getBuggyTrees() { - return buggyTrees; - } - - public String getSizes() { - return sizes; - } - - public String getTokensOfSourceCode() { - return tokensOfSourceCode; - } - - public String getOriginalTree() { - return originalTree; - } - - public String getActionSets() { - return actionSets; - } } diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java index e34c453..5a63237 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java @@ -4,28 +4,10 @@ import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.StringReader; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; -import org.eclipse.jdt.core.dom.CompilationUnit; - -import com.github.gumtreediff.actions.model.Action; -import com.github.gumtreediff.actions.model.Move; -import com.github.gumtreediff.actions.model.Update; -import com.github.gumtreediff.tree.ITree; - -import edu.lu.uni.serval.FixPattern.utils.Checker; -import edu.lu.uni.serval.FixPatternParser.CUCreator; -import edu.lu.uni.serval.FixPatternParser.Tokenizer; -import edu.lu.uni.serval.config.Configuration; -import edu.lu.uni.serval.gumtree.GumTreeComparer; -import edu.lu.uni.serval.gumtree.regroup.ActionFilter; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; -import edu.lu.uni.serval.gumtree.regroup.SimpleTree; -import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; +import edu.lu.uni.serval.FixPatternParser.Parser; import edu.lu.uni.serval.utils.FileHelper; /** @@ -34,143 +16,13 @@ import edu.lu.uni.serval.utils.FileHelper; * @author kui.liu * */ -public class FixedViolationParser { +public class FixedViolationParser extends Parser { - private String astEditScripts = ""; // it will be used for fix patterns mining. - private String patchesSourceCode = ""; // testing - private String buggyTrees = ""; // Compute similarity for bug localization. - private String sizes = ""; // fix patterns' selection before mining. - private String tokensOfSourceCode = ""; // Compute similarity for bug localization. - private String originalTree = ""; // Guide of generating patches. - private String actionSets = ""; // Guide of generating patches. - + @Override public void parseFixPatterns(File prevFile, File revFile, File positionFile) { - // GumTree results - List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); - - if (gumTreeResults != null && gumTreeResults.size() > 0) { - List allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults); - // Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part. - // TODO: variable effects range, sub-actions are these kinds of modification? - List actionSets = new ActionFilter().filterOutUselessActions(allActionSets); - - if (actionSets.size() > 0) { - // DiffEntry Hunks: filter out big hunks. - Map positions = readPositions(positionFile); - for (HierarchicalActionSet actionSet : actionSets) { - // position of buggy statements - int startPosition = 0; - int endPosition = 0; - // position of fixed statements - int startPosition2 = 0; - int endPosition2 = 0; - - String actionStr = actionSet.getActionString(); - String astNodeType = actionSet.getAstNodeType(); - if (actionStr.startsWith("INS")) { - startPosition2 = actionSet.getStartPosition(); - endPosition2 = startPosition2 + actionSet.getLength(); - List firstAndLastMov = getFirstAndLastMoveAction(actionSet); - if (firstAndLastMov != null) { - startPosition = firstAndLastMov.get(0).getNode().getPos(); - ITree lastTree = firstAndLastMov.get(1).getNode(); - endPosition = lastTree.getPos() + lastTree.getLength(); - } else { // Ignore the pure insert actions without any move actions. - continue; - } - } else if (actionStr.startsWith("UPD")) { - startPosition = actionSet.getStartPosition(); - endPosition = startPosition + actionSet.getLength(); - Update update = (Update) actionSet.getAction(); - ITree newNode = update.getNewNode(); - startPosition2 = newNode.getPos(); - endPosition2 = startPosition2 + newNode.getLength(); - - if (Checker.containsBodyBlock(astNodeType)) { - List children = update.getNode().getChildren(); - endPosition = getEndPosition(children); - List newChildren = newNode.getChildren(); - endPosition2 = getEndPosition(newChildren); - - if (endPosition == 0) { - endPosition = startPosition + actionSet.getLength(); - } - if (endPosition2 == 0) { - endPosition2 = startPosition2 + newNode.getLength(); - } - } - } else {// DEL actions and MOV actions: we don't need these actions, as for now. - continue; - } - if (startPosition == 0 || startPosition2 == 0) { - continue; - } - - CUCreator cuCreator = new CUCreator(); - CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); - CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); - if (prevUnit == null || revUnit == null) { - continue; - } - - // Get line numbers. - int startLine = prevUnit.getLineNumber(startPosition); - int endLine = prevUnit.getLineNumber(endPosition); - int startLine2 = revUnit.getLineNumber(startPosition2); - int endLine2 = revUnit.getLineNumber(endPosition2); - - if (!inPositions(startLine, endLine, positions)) { - continue; - } -// if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue; - - /* - * Convert the ITree of buggy code to a simple tree. - * It will be used to compute the similarity. - */ - SimplifyTree abstractIdentifier = new SimplifyTree(); - abstractIdentifier.abstractTree(actionSet); - SimpleTree simpleTree = actionSet.getSimpleTree(); - if (simpleTree == null) { // Failed to get the simple tree for INS actions. - continue; - } - - /** - * Select edit scripts for deep learning. - * Edit scripts will be used to mine common fix patterns. - */ - // 1. First level: AST node type. - String astEditScripts = getASTEditScripts(actionSet); - int size = astEditScripts.split(" ").length; - if (size == 1) { - continue; - } - - // Source Code of patches. - String patchSourceCode = getPatchSourceCode(prevFile, revFile, startLine, endLine, startLine2, endLine2); - this.patchesSourceCode += Configuration.PATCH_SIGNAL + "\n" + revFile.getName() + "\n" + patchSourceCode + "\n"; - this.sizes += size + "\n"; - this.astEditScripts += astEditScripts + "\n"; - // 2. source code: raw tokens - String rawTokenEditScripts = getRawTokenEditScripts(actionSet); - // 3. abstract identifiers: - String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet); - // 4. semi-source code: - String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet); - -// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n"; - this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n"; -// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n"; -// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n"; - } - actionSets.clear(); - } - allActionSets.clear(); - gumTreeResults.clear(); - } } - private boolean inPositions(int startLine, int endLine, Map positions) { + protected boolean inPositions(int startLine, int endLine, Map positions) { for (Map.Entry entry : positions.entrySet()) { int startPosi = entry.getKey(); int endPosi = entry.getValue(); @@ -179,7 +31,7 @@ public class FixedViolationParser { return false; } - private Map readPositions(File positionFile) { + protected Map readPositions(File positionFile) { Map positions = new HashMap<>(); String fileContent = FileHelper.readFile(positionFile); BufferedReader reader = null; @@ -204,91 +56,14 @@ public class FixedViolationParser { return positions; } - private List getFirstAndLastMoveAction(HierarchicalActionSet gumTreeResult) { - List firstAndLastMoveActions = new ArrayList<>(); - List actions = gumTreeResult.getSubActions(); - if (actions.size() == 0) { - return null; - } - Move firstMoveAction = null; - Move lastMoveAction = null; - while (actions.size() > 0) { - List subActions = new ArrayList<>(); - for (HierarchicalActionSet action : actions) { - subActions.addAll(action.getSubActions()); - if (action.toString().startsWith("MOV")) { - if (firstMoveAction == null) { - firstMoveAction = (Move) action.getAction(); - lastMoveAction = (Move) action.getAction(); - } else { - int startPosition = action.getStartPosition(); - int length = action.getLength(); - int startPositionFirst = firstMoveAction.getPosition(); - int startPositionLast = lastMoveAction.getPosition(); - int lengthLast = lastMoveAction.getNode().getLength(); - if (startPosition < startPositionFirst || (startPosition == startPositionFirst && length > firstMoveAction.getLength())) { - firstMoveAction = (Move) action.getAction(); - } - if ((startPosition + length) > (startPositionLast + lengthLast)) { - lastMoveAction = (Move) action.getAction(); - } - } - } - } - - actions.clear(); - actions.addAll(subActions); - } - if (firstMoveAction == null) { - return null; - } - firstAndLastMoveActions.add(firstMoveAction); - firstAndLastMoveActions.add(lastMoveAction); - return firstAndLastMoveActions; - } - - private String readActionSet(HierarchicalActionSet actionSet, String line) { - String str = line + actionSet.getActionString() + "\n"; - List subActions = actionSet.getSubActions(); - for (HierarchicalActionSet subAction : subActions) { - str += readActionSet(subAction, line + "---"); - } - return str; - } - private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) { - // TODO Auto-generated method stub - return null; - } - - private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) { - // TODO Auto-generated method stub - return null; - } - - private String getRawTokenEditScripts(HierarchicalActionSet actionSet) { - // TODO Auto-generated method stub - return null; - } - - private int getEndPosition(List children) { - int endPosition = 0; - for (ITree child : children) { - if (child.getLabel().endsWith("Body")) { - endPosition = child.getPos() - 1; - break; - } - } - return endPosition; - } - - private String getPatchSourceCode(File prevFile, File revFile, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) { + protected String getPatchSourceCode(File prevFile, File revFile, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) { String buggyStatements = readSourceCode(prevFile, startLineNum, endLineNum, "-"); String fixedStatements = readSourceCode(revFile, startLineNum2, endLineNum2, "+"); return buggyStatements + fixedStatements; } - private String readSourceCode(File file, int startLineNum, int endLineNum, String type) { + protected String readSourceCode(File file, int startLineNum, int endLineNum, String type) { String sourceCode = ""; String fileContent = FileHelper.readFile(file); BufferedReader reader = null; @@ -315,76 +90,4 @@ public class FixedViolationParser { return sourceCode; } - /** - * Get the AST node based edit script of patches in terms of breadth first. - * - * @param actionSet - * @return - */ - private String getASTEditScripts(HierarchicalActionSet actionSet) { - String editScript = ""; - - List actionSets = new ArrayList<>(); - actionSets.add(actionSet); - while (actionSets.size() != 0) { - List subSets = new ArrayList<>(); - for (HierarchicalActionSet set : actionSets) { - subSets.addAll(set.getSubActions()); - String actionStr = set.getActionString(); - int index = actionStr.indexOf("@@"); - String singleEdit = actionStr.substring(0, index).replace(" ", ""); - - if (singleEdit.endsWith("SimpleName")) { - actionStr = actionStr.substring(index + 2); - if (actionStr.startsWith("MethodName")) { - singleEdit = singleEdit.replace("SimpleName", "MethodName"); - } else { - if (actionStr.startsWith("Name")) { - actionStr = actionStr.substring(5, 6); - if (!actionStr.equals(actionStr.toLowerCase())) { - singleEdit = singleEdit.replace("SimpleName", "Name"); - } else { - singleEdit = singleEdit.replace("SimpleName", "Variable"); - } - } else { - singleEdit = singleEdit.replace("SimpleName", "Variable"); - } - } - } - - editScript += singleEdit + " "; - } - actionSets.clear(); - actionSets.addAll(subSets); - } - return editScript; - } - - public String getAstEditScripts() { - return astEditScripts; - } - - public String getPatchesSourceCode() { - return patchesSourceCode; - } - - public String getBuggyTrees() { - return buggyTrees; - } - - public String getSizes() { - return sizes; - } - - public String getTokensOfSourceCode() { - return tokensOfSourceCode; - } - - public String getOriginalTree() { - return originalTree; - } - - public String getActionSets() { - return actionSets; - } } diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java new file mode 100644 index 0000000..20b3d11 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java @@ -0,0 +1,158 @@ +package edu.lu.uni.serval.FixPatternParser.violations; + +import java.io.File; +import java.util.List; +import java.util.Map; + +import org.eclipse.jdt.core.dom.CompilationUnit; + +import com.github.gumtreediff.actions.model.Move; +import com.github.gumtreediff.actions.model.Update; +import com.github.gumtreediff.tree.ITree; + +import edu.lu.uni.serval.FixPattern.utils.Checker; +import edu.lu.uni.serval.FixPatternParser.CUCreator; +import edu.lu.uni.serval.FixPatternParser.Tokenizer; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.gumtree.regroup.SimpleTree; +import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; + +/** + * Parse fix patterns with GumTree. + * + * @author kui.liu + * + */ +public class FixedViolationSingleStatementParser extends FixedViolationParser { + + @Override + public void parseFixPatterns(File prevFile, File revFile, File positionFile) { + // GumTree results + List actionSets = parseChangedSourceCodeWithGumTree(prevFile, revFile); + + if (actionSets.size() > 0) { + // DiffEntry Hunks: filter out big hunks. + Map positions = readPositions(positionFile); + for (HierarchicalActionSet actionSet : actionSets) { + // position of buggy statements + int startPosition = 0; + int endPosition = 0; + // position of fixed statements + int startPosition2 = 0; + int endPosition2 = 0; + + String actionStr = actionSet.getActionString(); + String astNodeType = actionSet.getAstNodeType(); + if (actionStr.startsWith("INS")) { + startPosition2 = actionSet.getStartPosition(); + endPosition2 = startPosition2 + actionSet.getLength(); + List firstAndLastMov = getFirstAndLastMoveAction(actionSet); + if (firstAndLastMov != null) { + startPosition = firstAndLastMov.get(0).getNode().getPos(); + ITree lastTree = firstAndLastMov.get(1).getNode(); + endPosition = lastTree.getPos() + lastTree.getLength(); + } else { // Ignore the pure insert actions without any move actions. + continue; + } + } else if (actionStr.startsWith("UPD")) { + startPosition = actionSet.getStartPosition(); + endPosition = startPosition + actionSet.getLength(); + Update update = (Update) actionSet.getAction(); + ITree newNode = update.getNewNode(); + startPosition2 = newNode.getPos(); + endPosition2 = startPosition2 + newNode.getLength(); + + if (Checker.containsBodyBlock(astNodeType)) { + List children = update.getNode().getChildren(); + endPosition = getEndPosition(children); + List newChildren = newNode.getChildren(); + endPosition2 = getEndPosition(newChildren); + + if (endPosition == 0) { + endPosition = startPosition + actionSet.getLength(); + } + if (endPosition2 == 0) { + endPosition2 = startPosition2 + newNode.getLength(); + } + } + } else if (actionStr.startsWith("MOV")) {// DEL actions and MOV actions: we don't need these actions, as for now. + continue; +// startPosition = actionSet.getStartPosition(); +// endPosition = startPosition + actionSet.getLength(); +// ITree node = actionSet.getNode().getParent(); +// startPosition2 = node.getPos(); +// endPosition2 = startPosition2 + node.getLength(); + } else { + startPosition = actionSet.getStartPosition(); + endPosition = startPosition + actionSet.getLength(); + continue; + } + if (startPosition == 0 || startPosition2 == 0) { + continue; + } + + CUCreator cuCreator = new CUCreator(); + CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); + CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); + if (prevUnit == null || revUnit == null) { + continue; + } + + // Get line numbers. + int startLine = prevUnit.getLineNumber(startPosition); + int endLine = prevUnit.getLineNumber(endPosition); + int startLine2 = revUnit.getLineNumber(startPosition2); + int endLine2 = revUnit.getLineNumber(endPosition2); + + if (!inPositions(startLine, endLine, positions)) { + continue; + } + if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue; + + /* + * Convert the ITree of buggy code to a simple tree. + * It will be used to compute the similarity. + */ + SimplifyTree abstractIdentifier = new SimplifyTree(); + abstractIdentifier.abstractTree(actionSet); + SimpleTree simpleTree = actionSet.getSimpleTree(); + if (simpleTree == null) { // Failed to get the simple tree for INS actions. + continue; + } + + /** + * Select edit scripts for deep learning. + * Edit scripts will be used to mine common fix patterns. + */ + // 1. First level: AST node type. + String astEditScripts = getASTEditScripts(actionSet); + int size = astEditScripts.split(" ").length; + if (size == 1) { + continue; + } + + // Source Code of patches. + String patchSourceCode = getPatchSourceCode(prevFile, revFile, startLine, endLine, startLine2, endLine2); + this.patchesSourceCode += Configuration.PATCH_SIGNAL + "\n" + revFile.getName() + "\n" + patchSourceCode + "\n"; + this.sizes += size + "\n"; + this.astEditScripts += astEditScripts + "\n"; + // 2. source code: raw tokens + String rawTokenEditScripts = getRawTokenEditScripts(actionSet); + // 3. abstract identifiers: + String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet); + // 4. semi-source code: + String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet); + +// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n"; + this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n"; +// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n"; +// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n"; + } + actionSets.clear(); + } else { + System.out.println(1); + } + } + +} diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java index 35d005c..1bc765b 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java @@ -18,7 +18,7 @@ import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; import edu.lu.uni.serval.FixPatternParser.RunnableParser; -import edu.lu.uni.serval.FixPatternParser.SingleStatementParser; +import edu.lu.uni.serval.FixPatternParser.patch.CommitPatchParser; import edu.lu.uni.serval.config.Configuration; import edu.lu.uni.serval.utils.FileHelper; @@ -67,7 +67,7 @@ public class ParseFixPatternWorker extends UntypedActor { File revFile = msgFile.getRevFile(); File prevFile = msgFile.getPrevFile(); File diffentryFile = msgFile.getDiffEntryFile(); - SingleStatementParser parser = new SingleStatementParser(); + CommitPatchParser parser = new CommitPatchParser(); final ExecutorService executor = Executors.newSingleThreadExecutor(); // schedule the work diff --git a/src/main/java/edu/lu/uni/serval/violation/parse/ViolationParser.java b/src/main/java/edu/lu/uni/serval/violation/parse/ViolationParser.java index 07a7f0f..f209c37 100644 --- a/src/main/java/edu/lu/uni/serval/violation/parse/ViolationParser.java +++ b/src/main/java/edu/lu/uni/serval/violation/parse/ViolationParser.java @@ -47,7 +47,7 @@ public class ViolationParser { public void parseViolations(String fixedAlarmFile, List repos) { AlarmsReader reader = new AlarmsReader(); Map violations = reader.readAlarmsList(fixedAlarmFile); - + int a = 0; for (Map.Entry entry : violations.entrySet()) { String projectName = entry.getKey(); String repoName = ""; @@ -57,7 +57,10 @@ public class ViolationParser { break; } } - if ("".equals(repoName)) continue; + if ("".equals(repoName)) { + a ++; + continue; + } Violation violation = entry.getValue(); List alarms = violation.getAlarms(); @@ -100,6 +103,7 @@ public class ViolationParser { gitRepo.close(); } } + System.out.println(a); } private String readPosition(Map positions) {