diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step1.java b/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step1.java index 898e555..605cd1c 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step1.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step1.java @@ -16,7 +16,7 @@ public class Step1 { public static void main(String[] args) { String editScriptsFile = Configuration.EDITSCRIPT_SIZES_FILE; String patchesSourceCodeFile = Configuration.PATCH_SOURCECODE_FILE; - String buggyTokensFile = Configuration.BUGGY_CODY_TOKENS_FILE; + String buggyTokensFile = Configuration.BUGGY_CODE_TOKENS_FILE; String editScriptSizesFile = Configuration.EDITSCRIPT_SIZES_FILE; FileHelper.deleteFile(editScriptsFile); FileHelper.deleteFile(patchesSourceCodeFile); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step8.java b/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step8.java index e1bd185..4f7664d 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step8.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternMining/App/Step8.java @@ -2,8 +2,8 @@ package edu.lu.uni.serval.FixPatternMining.App; import java.io.File; +import edu.lu.uni.serval.bugLocalization.ProjectScanner; import edu.lu.uni.serval.config.Configuration; -import edu.lu.uni.serval.evaluation.ProjectScanner; import edu.lu.uni.serval.utils.FileHelper; /** diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMining/DataPrepare/DataPreparation.java b/src/main/java/edu/lu/uni/serval/FixPatternMining/DataPrepare/DataPreparation.java index ce03dcf..d5edbab 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternMining/DataPrepare/DataPreparation.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternMining/DataPrepare/DataPreparation.java @@ -37,7 +37,7 @@ public class DataPreparation { String editScriptsFile = Configuration.EDITSCRIPT_SIZES_FILE; String patchesSourceCodeFile = Configuration.PATCH_SOURCECODE_FILE; - String buggyTokensFile = Configuration.BUGGY_CODY_TOKENS_FILE; + String buggyTokensFile = Configuration.BUGGY_CODE_TOKENS_FILE; String editScriptSizesFile = Configuration.EDITSCRIPT_SIZES_FILE; File file = new File(editScriptsFilePath); File[] subFiles = file.listFiles(); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/HunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/HunkParser.java index 021cb94..72a16e5 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/HunkParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/HunkParser.java @@ -2,7 +2,6 @@ package edu.lu.uni.serval.FixPatternParser; import java.io.BufferedReader; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; @@ -38,7 +37,7 @@ public class HunkParser { private String originalTree = ""; // Guide of generating patches. private String actionSets = ""; // Guide of generating patches. - public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) throws FileNotFoundException, IOException { + public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { // GumTree results List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/SingleStatementParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/SingleStatementParser.java index 9082454..0b4d202 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/SingleStatementParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/SingleStatementParser.java @@ -2,7 +2,6 @@ package edu.lu.uni.serval.FixPatternParser; import java.io.BufferedReader; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; @@ -42,7 +41,7 @@ public class SingleStatementParser { private String originalTree = ""; // Guide of generating patches. private String actionSets = ""; // Guide of generating patches. - public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) throws FileNotFoundException, IOException { + public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { // GumTree results List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java new file mode 100644 index 0000000..d268fe0 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java @@ -0,0 +1,283 @@ +package edu.lu.uni.serval.FixPatternParser.violations; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +import com.github.gumtreediff.actions.model.Action; + +import edu.lu.uni.serval.FixPatternParser.Tokenizer; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.diffentry.DiffEntryHunk; +import edu.lu.uni.serval.diffentry.DiffEntryReader; +import edu.lu.uni.serval.gumtree.GumTreeComparer; +import edu.lu.uni.serval.gumtree.regroup.ActionFilter; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; +import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter; +import edu.lu.uni.serval.gumtree.regroup.HunkFixPattern; +import edu.lu.uni.serval.gumtree.regroup.SimpleTree; +import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; + +/** + * Parse fix patterns with GumTree. + * + * @author kui.liu + * + */ +public class FixedViolationHunkParser { + + private String astEditScripts = ""; // it will be used for fix patterns mining. + private String patchesSourceCode = ""; // testing + private String buggyTrees = ""; // Compute similarity for bug localization. + private String sizes = ""; // fix patterns' selection before mining. + private String tokensOfSourceCode = ""; // Compute similarity for bug localization. + private String originalTree = ""; // Guide of generating patches. + private String actionSets = ""; // Guide of generating patches. + + public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { + + // GumTree results + List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); + + if (gumTreeResults != null && gumTreeResults.size() > 0) { + List actionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults); + + ActionFilter filter = new ActionFilter(); + // Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part. + // TODO: variable effects range, sub-actions are these kinds of modification? + List allActionSets = filter.filterOutUselessActions(actionSets); + if (allActionSets.size() == 0) return; + // DiffEntry size: filter out big hunks. + List diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile); + //Filter out the modify actions, which are not in the DiffEntry hunks. + HunkActionFilter hunkFilter = new HunkActionFilter(); + List allHunkFixPatternss = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, allActionSets, revFile, prevFile); + + for (HunkFixPattern hunkFixPattern : allHunkFixPatternss) { + // Range of buggy source code + int startLine = 0; + int endLine = 0; + // Range of fixing source code + int startLine2 = 0; + int endLine2 = 0; + /* + * Convert the ITree of buggy code to a simple tree. + * It will be used to compute the similarity. + */ + List hunkActionSets = hunkFixPattern.getHunkActionSets(); + SimpleTree simpleTree = new SimpleTree(); + simpleTree.setLabel("Block"); + simpleTree.setNodeType("Block"); + List children = new ArrayList<>(); + String astEditScripts = ""; + for (HierarchicalActionSet hunkActionSet : hunkActionSets) { + SimplifyTree abstractIdentifier = new SimplifyTree(); + abstractIdentifier.abstractTree(hunkActionSet); + SimpleTree simpleT = hunkActionSet.getSimpleTree(); + if (simpleT == null) { // Failed to get the simple tree for INS actions. + continue; + } + children.add(simpleT); + + /** + * Select edit scripts for deep learning. + * Edit scripts will be used to mine common fix patterns. + */ + // 1. First level: AST node type. + astEditScripts += getASTEditScripts(hunkActionSet); + // 2. source code: raw tokens + // 3. abstract identifiers: + // 4. semi-source code: + + if (startLine == 0) { + startLine = hunkActionSet.getBugStartLineNum(); + endLine = hunkActionSet.getBugEndLineNum(); + startLine2 = hunkActionSet.getFixStartLineNum(); + endLine2 = hunkActionSet.getFixEndLineNum(); + } else { + if (startLine > hunkActionSet.getBugStartLineNum()) startLine = hunkActionSet.getBugStartLineNum(); + if (startLine2 > hunkActionSet.getFixStartLineNum()) startLine2 = hunkActionSet.getFixStartLineNum(); + if (endLine < hunkActionSet.getBugEndLineNum()) endLine = hunkActionSet.getBugEndLineNum(); + if (endLine2 < hunkActionSet.getFixEndLineNum()) endLine2 = hunkActionSet.getFixEndLineNum(); + } + } + if (children.size() == 0) continue; + if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue; + + simpleTree.setChildren(children); + simpleTree.setParent(null); + + // Source Code of patches. + String patchSourceCode = getPatchSourceCode(hunkFixPattern.getHunk(), startLine, endLine, startLine2, endLine2); + if ("".equals(patchSourceCode)) continue; + + this.patchesSourceCode += "PATCH###\n" + patchSourceCode + "\n"; + int size = astEditScripts.split(" ").length; + this.sizes += size + "\n"; + this.astEditScripts += astEditScripts + "\n"; + +// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n"; + this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n"; +// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n"; +// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n"; + + } + } + } + + private String readActionSet(HierarchicalActionSet actionSet, String line) { + String str = line + actionSet.getActionString() + "\n"; + List subActions = actionSet.getSubActions(); + for (HierarchicalActionSet subAction : subActions) { + str += readActionSet(subAction, line + "---"); + } + return str; + } + + private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) { + // TODO Auto-generated method stub + return null; + } + + private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) { + // TODO Auto-generated method stub + return null; + } + + private String getRawTokenEditScripts(HierarchicalActionSet actionSet) { + // TODO Auto-generated method stub + return null; + } + + private String getPatchSourceCode(DiffEntryHunk hunk, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) { + String sourceCode = hunk.getHunk(); + int bugStartLine = hunk.getBugLineStartNum(); + int fixStartLine = hunk.getFixLineStartNum();String buggyStatements = ""; + String fixedStatements = ""; + BufferedReader reader = null; + try { + reader = new BufferedReader(new StringReader(sourceCode)); + String line = null; + int bugLines = 0; + int fixLines = 0; + int contextLines = 0; // counter of non-buggy code line. + while ((line = reader.readLine()) != null) { + int bugLineIndex = bugLines + contextLines; + int fixLineIndex = fixLines + contextLines; + if (line.startsWith("-")) { + if (bugStartLine + bugLineIndex >= startLineNum && bugStartLine + bugLineIndex <= endLineNum) { + buggyStatements += line + "\n"; + } + bugLines ++; + } else if (line.startsWith("+")) { + if (fixStartLine + fixLineIndex >= startLineNum2 && fixStartLine + fixLineIndex <= endLineNum2) { + fixedStatements += line + "\n"; + } + fixLines ++; + } else { + contextLines ++; + } + + if (bugStartLine + bugLineIndex > endLineNum && fixStartLine + fixLineIndex > endLineNum2) { + break; + } + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + if (reader != null) { + reader.close(); + reader = null; + } + } catch (IOException e) { + e.printStackTrace(); + } + } + return buggyStatements + fixedStatements; + } + + /** + * Get the AST node based edit script of patches in terms of breadth first. + * + * @param actionSet + * @return + */ + private String getASTEditScripts(HierarchicalActionSet actionSet) { + String editScript = ""; + + List actionSets = new ArrayList<>(); + actionSets.add(actionSet); + while (actionSets.size() != 0) { + List subSets = new ArrayList<>(); + for (HierarchicalActionSet set : actionSets) { + subSets.addAll(set.getSubActions()); + String actionStr = set.getActionString(); + int index = actionStr.indexOf("@@"); + String singleEdit = actionStr.substring(0, index).replace(" ", ""); + + if (singleEdit.endsWith("SimpleName")) { + actionStr = actionStr.substring(index + 2); + if (actionStr.startsWith("MethodName")) { + singleEdit = singleEdit.replace("SimpleName", "MethodName"); + } else { + if (actionStr.startsWith("Name")) { + actionStr = actionStr.substring(5, 6); + if (!actionStr.equals(actionStr.toLowerCase())) { + singleEdit = singleEdit.replace("SimpleName", "Name"); + } else { + singleEdit = singleEdit.replace("SimpleName", "Variable"); + } + } else { + singleEdit = singleEdit.replace("SimpleName", "Variable"); + } + } + } + + editScript += singleEdit + " "; + } + actionSets.clear(); + actionSets.addAll(subSets); + } + return editScript; + } + + private void clearITree(HierarchicalActionSet actionSet) { + actionSet.getAction().setNode(null); + for (HierarchicalActionSet subActionSet : actionSet.getSubActions()) { + clearITree(subActionSet); + } + } + + public String getAstEditScripts() { + return astEditScripts; + } + + public String getPatchesSourceCode() { + return patchesSourceCode; + } + + public String getBuggyTrees() { + return buggyTrees; + } + + public String getSizes() { + return sizes; + } + + public String getTokensOfSourceCode() { + return tokensOfSourceCode; + } + + public String getOriginalTree() { + return originalTree; + } + + public String getActionSets() { + return actionSets; + } +} diff --git a/src/main/java/edu/lu/uni/serval/evaluation/ProjectScanner.java b/src/main/java/edu/lu/uni/serval/bugLocalization/ProjectScanner.java similarity index 99% rename from src/main/java/edu/lu/uni/serval/evaluation/ProjectScanner.java rename to src/main/java/edu/lu/uni/serval/bugLocalization/ProjectScanner.java index 0a34a4c..6ec6317 100644 --- a/src/main/java/edu/lu/uni/serval/evaluation/ProjectScanner.java +++ b/src/main/java/edu/lu/uni/serval/bugLocalization/ProjectScanner.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.evaluation; +package edu.lu.uni.serval.bugLocalization; import java.io.File; import java.util.ArrayList; diff --git a/src/main/java/edu/lu/uni/serval/config/Configuration.java b/src/main/java/edu/lu/uni/serval/config/Configuration.java index 4fca1e5..42c7f76 100644 --- a/src/main/java/edu/lu/uni/serval/config/Configuration.java +++ b/src/main/java/edu/lu/uni/serval/config/Configuration.java @@ -1,7 +1,7 @@ package edu.lu.uni.serval.config; public class Configuration { - private static final String ROOT_PATH = "../"; // The root path of all output data. + private static final String ROOT_PATH = "";//"../"; // The root path of all output data. public static final int HUNK_SIZE = 7; // The limitation of source code lines of each DiffEntry, which will be selected as training data. public static final String BUGGY_TREE_SIGNAL = "BUGGY_TREE###"; // The starting signal of the tree of buggy source code . @@ -21,7 +21,7 @@ public class Configuration { public static final String EDITSCRIPTS_FILE = GUM_TREE_OUTPUT + "editScripts.list"; public static final String PATCH_SOURCECODE_FILE = GUM_TREE_OUTPUT + "patchSourceCode.list"; public static final String BUGGYTREES_FILE = GUM_TREE_OUTPUT + "buggyTrees.list"; - public static final String BUGGY_CODY_TOKENS_FILE = GUM_TREE_OUTPUT + "tokens.list"; + public static final String BUGGY_CODE_TOKENS_FILE = GUM_TREE_OUTPUT + "tokens.list"; public static final String EDITSCRIPT_SIZES_FILE = GUM_TREE_OUTPUT + "editScriptSizes.list"; public static final int VECTOR_SIZE_OF_EMBEDED_TOKEN1 = 100; // tokens of edit scripts.