diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java index 132f305..e02a1a7 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java @@ -13,7 +13,6 @@ import edu.lu.uni.serval.gumtree.GumTreeComparer; import edu.lu.uni.serval.gumtree.regroup.ActionFilter; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; -import edu.lu.uni.serval.gumtree.regroup.SimpleTree; import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; /** diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java index e0045fa..72ad67d 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java @@ -1,9 +1,6 @@ package edu.lu.uni.serval.FixPatternParser.violations; -import java.io.BufferedReader; import java.io.File; -import java.io.IOException; -import java.io.StringReader; import java.util.ArrayList; import java.util.List; @@ -13,8 +10,6 @@ import edu.lu.uni.serval.FixPatternParser.Tokenizer; import edu.lu.uni.serval.config.Configuration; import edu.lu.uni.serval.diffentry.DiffEntryHunk; import edu.lu.uni.serval.diffentry.DiffEntryReader; -import edu.lu.uni.serval.gumtree.GumTreeGenerator; -import edu.lu.uni.serval.gumtree.GumTreeGenerator.GumTreeType; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter; import edu.lu.uni.serval.gumtree.regroup.SimpleTree; @@ -29,8 +24,6 @@ import edu.lu.uni.serval.violation.code.parser.ViolationSourceCodeTree; */ public class FixedViolationHunkParser extends FixedViolationParser { - private static final int THRESHOLD_LINE = 3; - public String testingInfo = ""; public int nullMappingGumTreeResult = 0; @@ -39,47 +32,31 @@ public class FixedViolationHunkParser extends FixedViolationParser { public int nullSourceCode = 0; public int nullMatchedDiffEntry = 0; public int testInfos = 0; - - public FixedViolationHunkParser() { - } - public FixedViolationHunkParser(File positionFile) { - setPositionFile(positionFile); - } - public String unfixedViolations = ""; @Override public void parseFixPatterns(File prevFile, File revFile, File diffentryFile) { - List allViolations = readViolations(prevFile, revFile); -// if (allViolations.size() >= 0) return; // GumTree results List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); if (this.resultType != 0) { - String type = ""; - if (this.resultType == 1) { - type = "#NullGumTreeResult:"; - } else if (this.resultType == 2) { - type = "#NoSourceCodeChange:"; - } else if (this.resultType == 3) { - type = "#NoStatementChange:"; - } - for (Violation v : allViolations) { - this.unfixedViolations += type + revFile.getName() + ":" + v.getStartLineNum() + ":" + v.getEndLineNum() + ":" + v.getViolationType() + "\n"; - } +// String type = ""; +// if (this.resultType == 1) { +// type = "#NullGumTreeResult:"; +// } else if (this.resultType == 2) { +// type = "#NoSourceCodeChange:"; +// } else if (this.resultType == 3) { +// type = "#NoStatementChange:"; +// } } else { List diffentryHunks = new DiffEntryReader().readHunks2(diffentryFile); - // Identify DiffEntry hunks by positions of violations. - List violations = identifyFixRangeHeuristically(allViolations, diffentryHunks, revFile); //Filter out the modify actions, which are not in the DiffEntry hunks. HunkActionFilter hunkFilter = new HunkActionFilter(); - List selectedViolations = hunkFilter.filterActionsByModifiedRange2(violations, actionSets, revFile, prevFile); - this.unfixedViolations += hunkFilter.unfixedViolations; - this.nullMappingGumTreeResult += violations.size() - selectedViolations.size(); + List selectedPatchHunks = hunkFilter.filterActionsByModifiedRange2(diffentryHunks, actionSets, revFile, prevFile); - for (Violation violation : selectedViolations) { - List hunkActionSets = violation.getActionSets(); + for (DiffEntryHunk patchHunk : selectedPatchHunks) { + List hunkActionSets = patchHunk.getActionSets(); // multiple UPD, and some UPD contain other UPD. removeOverlapperdUPD(hunkActionSets); @@ -117,43 +94,27 @@ public class FixedViolationHunkParser extends FixedViolationParser { } if (fixStartLine == 0 && bugStartLine == 0) { - this.unfixedViolations += "#WRONG: " + revFile.getName() + ":" + violation.getStartLineNum() + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; + this.unfixedViolations += "#WRONG: " + revFile.getName() + ":" + patchHunk.getBugLineStartNum() + ", " + patchHunk.getBuggyHunkSize() + "\n"; this.nullMappingGumTreeResult ++; continue; } if (fixStartLine == 0 && bugStartLine != 0) {// pure delete actions. -// this.pureDeletions ++; -// this.unfixedViolations += "#PureDeletion:" + revFile.getName() + ":" + violation.getStartLineNum() + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; // get the exact buggy code by violation's position. TODO later } // if (children.size() == 0) continue; boolean isPureInsert = false; - if (bugStartLine == 0 && violation.getBugStartLineNum() > 0) { - bugStartLine = violation.getStartLineNum(); - bugEndLine = violation.getEndLineNum(); + if (bugStartLine == 0 && patchHunk.getBugLineStartNum() > 0) { + bugStartLine = patchHunk.getBugLineStartNum(); + bugEndLine = bugStartLine + patchHunk.getBuggyHunkSize() - 1; isPureInsert = true; // continue; } if ((bugEndLine - bugStartLine > Configuration.HUNK_SIZE && !isPureInsert) || fixEndLine - fixStartLine > Configuration.HUNK_SIZE) { -// this.largeHunk ++; -// this.unfixedViolations += "#LargeHunk:" + revFile.getName() + ":" + violation.getStartLineNum() -// + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; // continue; } - // Obtain Source Code of patches from DiffEntries. - String patchSourceCode = this.readPatchSourceCode(violation, bugStartLine, bugEndLine, fixStartLine, fixEndLine); - if ("".equals(patchSourceCode)) { - patchSourceCode = getPatchSourceCode(prevFile, revFile, bugStartLine, bugEndLine, fixStartLine, fixEndLine, isPureInsert); - if ("".equals(patchSourceCode)) { - this.nullSourceCode ++; - this.unfixedViolations += "#NullSourceCode:" + revFile.getName() + ":" + violation.getStartLineNum() - + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; - continue; - } - } /** * Select edit scripts for deep learning. @@ -162,8 +123,7 @@ public class FixedViolationHunkParser extends FixedViolationParser { // 1. First level: AST node type. String astEditScripts = getASTEditScriptsDeepFirst(hunkActionSets, bugEndPosition, fixEndPosition); if (astEditScripts.contains("\n") || astEditScripts.split(" ").length % 3 != 0) { - System.err.println("===+++===: " + revFile.getName() + ":" +violation.getStartLineNum() + ":" + - violation.getEndLineNum() + ":" + violation.getViolationType()); + System.err.println("===+++===: " + revFile.getName() + ":" +patchHunk.getBugLineStartNum() + ", " + patchHunk.getBuggyHunkSize()); } // 2. source code: raw tokens // 3. abstract identifiers: @@ -171,141 +131,82 @@ public class FixedViolationHunkParser extends FixedViolationParser { String[] editScriptTokens = astEditScripts.split(" "); int size = editScriptTokens.length; if (size == 1) { -// if ("MOVMethodDeclaration".equals(astEditScripts)) { -// this.nullMappingGumTreeResult ++; -// this.unfixedViolations += "#NullMatchedGumTreeResult:" + revFile.getName() + ":" +violation.getStartLineNum() + ":" + -// violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; -// continue; -// } this.nullMappingGumTreeResult ++; - this.unfixedViolations += "#NullMatchedGumTreeResult1:" + revFile.getName() + ":" +violation.getStartLineNum() + ":" + - violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; + this.unfixedViolations += "#NullMatchedGumTreeResult1:" + revFile.getName() + ":" + patchHunk.getBugLineStartNum() + ", " + patchHunk.getBuggyHunkSize() + "\n"; continue; } - String alarmType = violation.getViolationType(); - String patchPosition = "\n" + revFile.getName() + "\nPosition: " + violation.getStartLineNum() + " --> " + violation.getEndLineNum() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n"; -// String patchPosition = "\n" + "Position: " + violation.getStartLineNum() + " --> " + violation.getEndLineNum() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n"; - String info = Configuration.PATCH_SIGNAL + "\nAlarm Type :" + alarmType + "\n" + patchPosition + patchSourceCode + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets, bugEndPosition, fixEndPosition) + "\n"; -// String patchPosition = "\n" + "Position: " + violation.getStartLineNum() + " --> " + violation.getEndLineNum() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n"; -// String info = Configuration.PATCH_SIGNAL + "\nAlarm Type :" + alarmType + "\n" + patchSourceCode + "\n\n";// + patchPosition + revFile.getName() + "\n"; + String patchPosition = "\n" + revFile.getName() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n"; +// String info = Configuration.PATCH_SIGNAL + "\n" + patchPosition + patchHunk.getHunk() + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets, bugEndPosition, fixEndPosition) + "\n"; + String info = Configuration.PATCH_SIGNAL + "\n" + patchPosition + patchHunk.getHunk() + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets) + "\n"; // if (noUpdate(editScriptTokens)) { -// -// if (!"SE_NO_SERIALVERSIONID".equals(alarmType)) { -// if (containsFiledDeclaration(hunkActionSets)) { -//// this.nullMappingGumTreeResult ++; -// this.testingInfo += info + revFile.getName() + "\n"; -// this.testingInfo += "#TestingInfo: " + revFile.getName() + ":" + violation.getStartLineNum() -// + ":" + violation.getEndLineNum() + ":" + violation.getViolationType(); -// this.unfixedViolations += "#TestingInfo:" + revFile.getName() + ":" + violation.getStartLineNum() -// + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; -// this.testInfos ++; -// continue; -// } -// } -// -// if ("UL_UNRELEASED_LOCK".equals(alarmType) || -// "SF_SWITCH_NO_DEFAULT".equals(alarmType) || -// "SF_SWITCH_FALLTHROUGH".equals(alarmType) || -// "SE_NO_SERIALVERSIONID".equals(alarmType) || -// "REC_CATCH_EXCEPTION".equals(alarmType) || -// "OS_OPEN_STREAM".equals(alarmType) || -// "NP_ALWAYS_NULL".equals(alarmType) || -// "IS2_INCONSISTENT_SYNC".equals(alarmType) || -// "EI_EXPOSE_REP".equals(alarmType) ) { -// -// } else if (containSpecialStmt(hunkActionSets, bugEndPosition, fixEndPosition)) { -// -// } else { -//// this.nullMappingGumTreeResult ++; -// this.testingInfo += info + revFile.getName() + "\n"; -// this.testingInfo += "#TestingInfo: " + revFile.getName() + ":" + violation.getStartLineNum() -// + ":" + violation.getEndLineNum() + ":" + violation.getViolationType(); -// this.unfixedViolations += "#TestingInfo:" + revFile.getName() + ":" + violation.getStartLineNum() -// + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; -// this.testInfos ++; -// continue; -// } // } this.patchesSourceCode += info; this.sizes += size + "\n"; this.astEditScripts += astEditScripts + "\n"; - this.violationTypes += alarmType + "\n"; -// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n"; - SimpleTree simpleTree = getBuggyCodeTree(violation, bugEndPosition, prevFile, bugStartLine, bugEndLine); + SimpleTree simpleTree = getBuggyCodeTree(patchHunk, bugEndPosition, prevFile, bugStartLine, bugEndLine); String tokens = Tokenizer.getTokensDeepFirst(simpleTree).trim(); - if ("Block Block".equals(tokens)) { - tokens = getContextTokens(patchSourceCode); - } this.tokensOfSourceCode += tokens + "\n"; -// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n"; -// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n"; - } } } - private SimpleTree getBuggyCodeTree(Violation violation, int bugEndPosition, File prevFile, int bugStartLine, int bugEndLine) { - SimpleTree simpleTree = null; + private String getAstEditScripts(List hunkActionSets) { + String scripts = ""; + for (HierarchicalActionSet hunkActionSet : hunkActionSets) { + scripts += hunkActionSet.toString() + "\n"; + } + return scripts; + } + + private SimpleTree getBuggyCodeTree(DiffEntryHunk patchHunk, int bugEndPosition, File prevFile, int bugStartLine, int bugEndLine) { + SimpleTree simpleTree = new SimpleTree(); + simpleTree.setLabel("Block"); + simpleTree.setNodeType("Block"); + List children = new ArrayList<>(); - String type = violation.getViolationType(); - if ("EQ_DOESNT_OVERRIDE_EQUALS".equals(type)|| "HE_EQUALS_USE_HASHCODE".equals(type) || "HE_INHERITS_EQUALS_USE_HASHCODE".equals(type)|| - "SE_NO_SUITABLE_CONSTRUCTOR".equals(type) || "RI_REDUNDANT_INTERFACES".equals(type) - ||"CN_IDIOM".equals(type)||"SE_NO_SERIALVERSIONID".equals(type) || "SE_COMPARATOR_SHOULD_BE_SERIALIZABLE".equals(type)) { - // Class name level, tokens - // classStartP <= vS <= vE <= classEndP - ViolationSourceCodeTree parser = new ViolationSourceCodeTree(prevFile, violation.getStartLineNum(), violation.getEndLineNum()); - ITree classNameTree = parser.getClassNameTokens(); - simpleTree = new SimplifyTree().canonicalizeSourceCodeTree(classNameTree, null); - } else { - simpleTree = new SimpleTree(); - simpleTree.setLabel("Block"); - simpleTree.setNodeType("Block"); - List children = new ArrayList<>(); - - int vStartLine = violation.getStartLineNum(); - int vEndLine = violation.getEndLineNum(); - if (bugStartLine < vStartLine && vEndLine < bugEndLine) { - ViolationSourceCodeTree parser = new ViolationSourceCodeTree(prevFile, violation.getStartLineNum(), violation.getEndLineNum()); - parser.extract(); - List matchedTrees = parser.getViolationSourceCodeTrees(); - if (matchedTrees.size() > 0) { - for (ITree matchedTree : matchedTrees) { - SimpleTree simpleT = new SimplifyTree().canonicalizeSourceCodeTree(matchedTree, simpleTree); - children.add(simpleT); - } - } - } - if (children.size() == 0) { - List hunkActionSets = violation.getActionSets(); - /* - * Convert the ITree of buggy code to a simple tree. - * It will be used to compute the similarity. - */ - for (HierarchicalActionSet hunkActionSet : hunkActionSets) { - // TODO simplify buggy tree with buggy code. - /** - * Select edit scripts for deep learning. - * Edit scripts will be used to mine common fix patterns. - */ -// // 1. First level: AST node type. -// // 2. source code: raw tokens -// // 3. abstract identifiers: -// // 4. semi-source code: - SimplifyTree abstractIdentifier = new SimplifyTree(); - abstractIdentifier.abstractTree(hunkActionSet, bugEndPosition); - SimpleTree simpleT = hunkActionSet.getSimpleTree(); - if (simpleT == null) { // Failed to get the simple tree for INS actions. - continue; - } + int vStartLine = patchHunk.getBugLineStartNum(); + int vEndLine = vStartLine + patchHunk.getBuggyHunkSize() + 1; + if (bugStartLine < vStartLine && vEndLine < bugEndLine) { + ViolationSourceCodeTree parser = new ViolationSourceCodeTree(prevFile, vStartLine, vEndLine); + parser.extract(); + List matchedTrees = parser.getViolationSourceCodeTrees(); + if (matchedTrees.size() > 0) { + for (ITree matchedTree : matchedTrees) { + SimpleTree simpleT = new SimplifyTree().canonicalizeSourceCodeTree(matchedTree, simpleTree); children.add(simpleT); } } - simpleTree.setChildren(children); - simpleTree.setParent(null); } + if (children.size() == 0) { + List hunkActionSets = patchHunk.getActionSets(); + /* + * Convert the ITree of buggy code to a simple tree. + * It will be used to compute the similarity. + */ + for (HierarchicalActionSet hunkActionSet : hunkActionSets) { + // TODO simplify buggy tree with buggy code. + /** + * Select edit scripts for deep learning. + * Edit scripts will be used to mine common fix patterns. + */ +// // 1. First level: AST node type. +// // 2. source code: raw tokens +// // 3. abstract identifiers: +// // 4. semi-source code: + SimplifyTree abstractIdentifier = new SimplifyTree(); + abstractIdentifier.abstractTree(hunkActionSet, bugEndPosition); + SimpleTree simpleT = hunkActionSet.getSimpleTree(); + if (simpleT == null) { // Failed to get the simple tree for INS actions. + continue; + } + children.add(simpleT); + } + } + simpleTree.setChildren(children); + simpleTree.setParent(null); return simpleTree; } @@ -344,170 +245,4 @@ public class FixedViolationHunkParser extends FixedViolationParser { actionSets.removeAll(overlappedUpdates); } - private List identifyFixRangeHeuristically(List violations, List diffentryHunks, File revFile) { - List selectedViolations = new ArrayList<>(); - for (Violation violation : violations) { - - int violationStartLineNum = violation.getBugStartLineNum(); - - if (violationStartLineNum > 0) { - int violationEndLineNum = violation.getEndLineNum(); - for (int index = 0, hunkListSize = diffentryHunks.size(); index < hunkListSize; index ++) { - DiffEntryHunk hunk = diffentryHunks.get(index); - int bugStartLine = hunk.getBugLineStartNum(); - int bugRange = hunk.getBugRange(); - if (violationStartLineNum > bugStartLine + bugRange - 1) continue; - if (violationEndLineNum < bugStartLine) break; - - int fixStartLine = hunk.getFixLineStartNum(); - String diffentry = hunk.getHunk(); - BufferedReader reader = new BufferedReader(new StringReader(diffentry)); - String line = null; - try { - int currentBugLine = bugStartLine - 1; - int currentFixLine = fixStartLine - 1; - int fixedLines = 0; - int bugS = 0; - int fixS = 0; - int fixE = 0; - int bugFixStartLine = 0; // the heuristic inferred fix start line of the corresponding bug - int bugFixEndLine = 0; // the heuristic inferred fix end line of the corresponding bug - boolean isFixRange= false; - while ((line = reader.readLine()) != null) { - if (line.startsWith("+")) { - currentFixLine ++; - fixedLines++; - if ((violationStartLineNum <= currentBugLine && currentBugLine <= violationEndLineNum) || - (violationStartLineNum <= bugS && bugS <= violationEndLineNum)) { - if (fixS == 0) fixS = currentFixLine; - if (bugFixStartLine == -1) bugFixStartLine = currentFixLine; - if (isFixRange) bugFixEndLine = currentFixLine; - fixE = currentFixLine; - } else { - fixedLines = 0; - bugS = 0; - } - } else if (line.startsWith("-")) { - currentBugLine ++; - if (bugS == 0 || fixS != 0) bugS = currentBugLine;// currentBugLine may be larger than the violation end line. - - // INS combined with DEL, UPD or MOV. - // Infer the fix range for a violation heuristically. - if (currentBugLine >= violation.getStartLineNum() && currentBugLine <= violation.getEndLineNum()) { - bugFixStartLine = -1; - } - if (currentBugLine <= violation.getEndLineNum() && bugFixStartLine != 0) { - isFixRange = true; - } - if (bugFixEndLine > 0) isFixRange = false; - if (bugFixStartLine <= 0) fixS = 0; - } else { - currentBugLine ++; - currentFixLine ++; - - // pure INS - // Infer the fix range for a violation heuristically. - if (currentBugLine == violation.getStartLineNum()) { - if (bugFixStartLine <= 0) { - if (fixedLines != 0) { - if (fixedLines > THRESHOLD_LINE) { - fixedLines = THRESHOLD_LINE;// insert 3 lines. - } - bugFixStartLine = currentFixLine - fixedLines; - if (bugFixStartLine < violationStartLineNum) { - bugFixStartLine = violationStartLineNum; - } - } - else bugFixStartLine = currentFixLine; - } - } - if (currentBugLine == violation.getEndLineNum()) { - if (bugFixStartLine != 0 && bugFixEndLine == 0) { - bugFixEndLine = currentFixLine + THRESHOLD_LINE; - if (bugFixEndLine > violationEndLineNum) { - bugFixEndLine = violationEndLineNum; - } - } - } - if (bugFixEndLine > 0) isFixRange = false; - } - - if ((violationStartLineNum <= currentBugLine && currentBugLine <= violationEndLineNum)) { - if (!violation.getHunks().contains(hunk)) - violation.getHunks().add(hunk); - } - } - if (violation.getFixStartLineNum() == 0) { - violation.setFixStartLineNum(fixS); - } - violation.setFixEndLineNum(fixE); - - if (violation.getBugFixStartLineNum() == 0 && bugFixStartLine > 0) { - violation.setBugFixStartLineNum(bugFixStartLine); - } - if (bugFixEndLine > 0) { - violation.setBugFixEndLineNum(bugFixEndLine); - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - reader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - - if (violation.getHunks().size() == 0) { - // This fixed violation cannot be matched with a DiffEntry, it is difficult to identify related source code change for it. - this.nullMatchedDiffEntry ++; - this.unfixedViolations += "#NullDiffEntry: " + revFile.getName() + " : " +violation.getStartLineNum() + " : " - + violation.getEndLineNum() + " : " + violation.getViolationType() + "\n"; - } else { - selectedViolations.add(violation); - } - } else { - selectedViolations.add(violation); - } - } - - return selectedViolations; - } - - private String getContextTokens(String patchSourceCode) { - BufferedReader reader = new BufferedReader(new StringReader(patchSourceCode)); - String line = null; - String context = ""; - try { - while ((line = reader.readLine()) != null) { - if (line.startsWith("+") || line.startsWith("-")) { - continue; - } - context += line + "\n"; - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - reader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - if ("".equals(context)) { - return "Block Block"; - } else { - ITree tree = new GumTreeGenerator().generateITreeForCodeBlock(context, GumTreeType.EXP_JDT); - if (tree == null) { - return "Block Block"; - } - SimpleTree simpleTree = new SimplifyTree().canonicalizeSourceCodeTree(tree, null); - String tokens = Tokenizer.getTokensDeepFirst(simpleTree).trim(); - return tokens; - } - } - } diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java index 4b68cdd..9d48ade 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java @@ -10,13 +10,11 @@ import java.util.List; import com.github.gumtreediff.actions.model.Action; import edu.lu.uni.serval.FixPatternParser.Parser; -import edu.lu.uni.serval.diffentry.DiffEntryHunk; import edu.lu.uni.serval.gumtree.GumTreeComparer; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; import edu.lu.uni.serval.utils.FileHelper; import edu.lu.uni.serval.utils.ListSorter; -import edu.lu.uni.serval.violation.code.parser.ViolationSourceCodeTree; /** * Parse fix patterns with GumTree. @@ -36,13 +34,7 @@ public class FixedViolationParser extends Parser { */ public int resultType = 0; - private File positionFile = null; protected String violationTypes = ""; -// protected List uselessViolations; - - public void setPositionFile(File positionFile) { - this.positionFile = positionFile; - } @Override public void parseFixPatterns(File prevFile, File revFile, File diffentryFile) { @@ -90,60 +82,6 @@ public class FixedViolationParser extends Parser { } } - protected List readViolations(File prevFile, File revFile) { - String fileName = revFile.getName(); - List violations = new ArrayList<>(); - String fileContent = FileHelper.readFile(positionFile); - BufferedReader reader = null; - reader = new BufferedReader(new StringReader(fileContent)); - String line = null; - try { - while ((line = reader.readLine()) != null) { - String[] positionStr = line.split(":"); - int startLine = Integer.parseInt(positionStr[1]); - int endLine = Integer.parseInt(positionStr[2]); - String violationType = positionStr[0]; - - Violation violation = new Violation(startLine, endLine, violationType); - violation.setFileName(fileName); - - if (startLine == -1) { - violation.setBugStartLineNum(0); - continue; - } - - /* - * Get the parent range of a violation. - * Read DiffEntries with this range to get the start line and end line of a violation. - */ - ViolationSourceCodeTree alarmTree = new ViolationSourceCodeTree(prevFile, startLine, endLine); - alarmTree.locateParentNode(violationType); - int violationStartLine = alarmTree.getViolationFinalStartLine(); - violation.setBugStartLineNum(violationStartLine); - if (violationStartLine > 0) {// 0: no source code, -1: range is too large, which contains several inner classes, methods or fields. - violation.setBugEndLineNum(alarmTree.getViolationFinalEndLine()); -// if (violationType.equals("SE_NO_SERIALVERSIONID")){ -// FileHelper.outputToFile("OUTPUT/list1.txt", line + ":" + revFile.getName() + "\n", true); -// } - } else { -// if (!violationType.equals("SE_NO_SERIALVERSIONID")) { -// FileHelper.outputToFile("OUTPUT/list.txt", line + ":" + revFile.getName() + "\n", true); -// } - } - violations.add(violation); - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - reader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - return violations; - } - /** * Read patch source code from buggy and fixed files. * @param prevFile @@ -193,60 +131,6 @@ public class FixedViolationParser extends Parser { return sourceCode; } - /** - * Read patch source code from diffentries. - * @param violation - * @param bugStartLine - * @param bugEndLine - * @param fixStartLine - * @param fixEndLine - * @return - */ - protected String readPatchSourceCode(Violation violation, int bugStartLine, int bugEndLine, int fixStartLine, int fixEndLine) { - String patch = ""; - List diffentries = violation.getHunks(); - for (DiffEntryHunk diffentry : diffentries) { - int currentBugLine = diffentry.getBugLineStartNum() - 1; - int currentFixLine = diffentry.getFixLineStartNum() - 1; - String sourceCode = diffentry.getHunk(); - - BufferedReader reader = new BufferedReader(new StringReader(sourceCode)); - String line = null; - - try { - while ((line = reader.readLine()) != null) { - if (line.startsWith("+")) { - currentFixLine ++; - if (fixStartLine <= currentFixLine && currentFixLine <= fixEndLine) { - patch += line + "\n"; - } - } else if (line.startsWith("-")) { - currentBugLine ++; - if (bugStartLine <= currentBugLine && currentBugLine <= bugEndLine) { - patch += line + "\n"; - } - } else { - currentFixLine ++; - currentBugLine ++; - if ((bugStartLine <= currentBugLine && currentBugLine <= bugEndLine) || - (fixStartLine <= currentFixLine && currentFixLine <= fixEndLine)) { - patch += line + "\n"; - } - } - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - reader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - return patch; - } - public String getAlarmTypes() { return violationTypes; } diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java index 390f597..6fe1e46 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java @@ -39,7 +39,6 @@ public class FixedViolationSingleStatementParser extends FixedViolationParser { } // Read the positions of checked violations - List violations = readViolations(prevFile, revFile); for (HierarchicalActionSet actionSet : actionSets) { // position of buggy statements int startPosition = 0; @@ -95,7 +94,7 @@ public class FixedViolationSingleStatementParser extends FixedViolationParser { int startLine2 = revUnit.getLineNumber(startPosition2); int endLine2 = revUnit.getLineNumber(endPosition2); - Violation violation = findViolation(startLine, endLine, violations); + Violation violation = null;//findViolation(startLine, endLine, violations); if (violation == null) continue; if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParser.java index a0aa1f8..a370e99 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParser.java @@ -54,7 +54,6 @@ public class TestHunkParser { // int counter = 0; for (MessageFile msgFile : msgFiles) { FixedViolationHunkParser parser = new FixedViolationHunkParser(); - parser.setPositionFile(msgFile.getPositionFile()); final ExecutorService executor = Executors.newSingleThreadExecutor(); // schedule the work diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/AkkaParser.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/AkkaParser.java index 4a9a5d4..e1fdeb2 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/AkkaParser.java +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/AkkaParser.java @@ -22,11 +22,28 @@ public class AkkaParser { private static Logger log = LoggerFactory.getLogger(AkkaParser.class); + /** + * Two parameters: + * First one: the root path of input data. + * Second one: the number of threads. + * Third one: the threshold of selecting patch hunks. + * @param args + */ @SuppressWarnings("deprecation") public static void main(String[] args) { + String inputRootPath = args[0]; + int numberOfWorkers = Integer.parseInt(args[1]); + int hunkThreshold = 0; + try { + hunkThreshold = Integer.parseInt(args[2]); + } catch (NumberFormatException e1) { + hunkThreshold = 10; + } + + Configuration.ROOT_PATH = inputRootPath; + Configuration.HUNK_SIZE = hunkThreshold; // input data log.info("Get the input data..."); -// final List msgFiles = getMessageFiles(); final List msgFiles = getMessageFiles(Configuration.GUM_TREE_INPUT); log.info("MessageFiles: " + msgFiles.size()); @@ -35,22 +52,19 @@ public class AkkaParser { final String patchesSourceCodeFilePath = Configuration.PATCH_SOURCECODE_FILE_PATH; final String buggyTokensFilePath = Configuration.BUGGY_CODE_TOKEN_FILE_PATH; final String editScriptSizesFilePath = Configuration.EDITSCRIPT_SIZES_FILE_PATH; - final String alarmTypesFilePath = Configuration.ALARM_TYPES_FILE_PATH; FileHelper.deleteDirectory(editScriptsFilePath); FileHelper.deleteDirectory(patchesSourceCodeFilePath); FileHelper.deleteDirectory(buggyTokensFilePath); FileHelper.deleteDirectory(editScriptSizesFilePath); - FileHelper.deleteDirectory(alarmTypesFilePath); ActorSystem system = null; ActorRef parsingActor = null; - int numberOfWorkers = 431; final WorkMessage msg = new WorkMessage(0, msgFiles); try { log.info("Akka begins..."); system = ActorSystem.create("Mining-FixPattern-System"); parsingActor = system.actorOf(ParseFixPatternActor.props(numberOfWorkers, editScriptsFilePath, - patchesSourceCodeFilePath, buggyTokensFilePath, editScriptSizesFilePath, alarmTypesFilePath), "mine-fix-pattern-actor"); + patchesSourceCodeFilePath, buggyTokensFilePath, editScriptSizesFilePath), "mine-fix-pattern-actor"); parsingActor.tell(msg, ActorRef.noSender()); } catch (Exception e) { system.shutdown(); @@ -59,47 +73,6 @@ public class AkkaParser { } - - /** - * Get bug commit-related files. - * - * @return - */ - public static List getMessageFiles() { - String inputPath = Configuration.GUM_TREE_INPUT; //DiffEntries prevFiles revFiles - File inputFileDirector = new File(inputPath); - File[] files = inputFileDirector.listFiles(); // project folders - log.info("Projects: " + files.length); - List msgFiles = new ArrayList<>(); - - for (File file : files) { - if (!file.isDirectory()) continue; -// if (!(file.getName().startsWith("k") || file.getName().startsWith("l"))) continue; - if (file.getName().startsWith("a") || file.getName().startsWith("b") - || file.getName().startsWith("c") || file.getName().startsWith("d") - || file.getName().startsWith("e") || file.getName().startsWith("f") - || file.getName().startsWith("g") || file.getName().startsWith("h") - ||file.getName().startsWith("h") || file.getName().startsWith("i") - || file.getName().startsWith("k") || file.getName().startsWith("l") - || file.getName().startsWith("j") || file.getName().startsWith("t")) continue; -// if (!file.getName().startsWith("j")) continue; - log.info("Project name: " + file.getName()); - String projectFolder = file.getPath(); - File revFileFolder = new File(projectFolder + "/revFiles/");// revised file folder - File[] revFiles = revFileFolder.listFiles(); - for (File revFile : revFiles) { - if (revFile.getName().endsWith(".java")) { - File prevFile = new File(projectFolder + "/prevFiles/prev_" + revFile.getName());// previous file - File diffentryFile = new File(projectFolder + "/DiffEntries/" + revFile.getName().replace(".java", ".txt")); // DiffEntry file - MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); - msgFiles.add(msgFile); - } - } - } - - return msgFiles; - } - /** * Get violation-related files. * diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternActor.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternActor.java index dc1b5af..746db72 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternActor.java +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternActor.java @@ -1,6 +1,5 @@ package edu.lu.uni.serval.MultipleThreadsParser; -import java.util.ArrayList; import java.util.List; import org.slf4j.Logger; @@ -21,15 +20,15 @@ public class ParseFixPatternActor extends UntypedActor { private int counter = 0; public ParseFixPatternActor(int numberOfWorkers, String editScriptsFilePath, String patchesSourceCodeFilePath, - String buggyTokensFilePath, String editScriptSizesFilePath, String alarmTypesFilePath) { + String buggyTokensFilePath, String editScriptSizesFilePath) { mineRouter = this.getContext().actorOf(new RoundRobinPool(numberOfWorkers) .props(ParseFixPatternWorker.props(editScriptsFilePath, patchesSourceCodeFilePath, - buggyTokensFilePath, editScriptSizesFilePath, alarmTypesFilePath)), "mine-fix-pattern-router"); + buggyTokensFilePath, editScriptSizesFilePath)), "mine-fix-pattern-router"); this.numberOfWorkers = numberOfWorkers; } public static Props props(final int numberOfWorkers, final String editScriptsFilePath, final String patchesSourceCodeFilePath, - final String buggyTokensFilePath, final String editScriptSizesFilePath, final String alarmTypesFilePath) { + final String buggyTokensFilePath, final String editScriptSizesFilePath) { return Props.create(new Creator() { @@ -38,7 +37,7 @@ public class ParseFixPatternActor extends UntypedActor { @Override public ParseFixPatternActor create() throws Exception { return new ParseFixPatternActor(numberOfWorkers, editScriptsFilePath, patchesSourceCodeFilePath, - buggyTokensFilePath, editScriptSizesFilePath, alarmTypesFilePath); + buggyTokensFilePath, editScriptSizesFilePath); } }); @@ -50,26 +49,25 @@ public class ParseFixPatternActor extends UntypedActor { if (message instanceof WorkMessage) { List files = ((WorkMessage) message).getMsgFiles(); int size = files.size(); - int average = (int) Math.round((double) size / numberOfWorkers); + int average = size / numberOfWorkers; + int reminder = size % numberOfWorkers; + int counter = 0; for (int i = 0; i < numberOfWorkers; i ++) { - int fromIndex = i * average; - int toIndex = (i + 1) * average; - if (i == numberOfWorkers - 1) { - toIndex = size; - } + int fromIndex = i * average + counter; + if (counter < reminder) counter ++; + int toIndex = (i + 1) * average + counter; - List filesOfWorkers = new ArrayList<>(); - filesOfWorkers.addAll(files.subList(fromIndex, toIndex)); + List filesOfWorkers = files.subList(fromIndex, toIndex); final WorkMessage workMsg = new WorkMessage(i + 1, filesOfWorkers); mineRouter.tell(workMsg, getSelf()); logger.info("Assign a task to worker #" + (i + 1) + "..."); } } else if ("STOP".equals(message.toString())) { counter ++; - logger.info(counter + " workers finished their work..."); + logger.info(counter + " workers finailized their work..."); if (counter >= numberOfWorkers) { - logger.info("All workers finished their work..."); + logger.info("All workers finailized their work..."); this.getContext().stop(mineRouter); this.getContext().stop(getSelf()); this.getContext().system().shutdown(); diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java index 2e2d5ae..f4982cf 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java @@ -31,19 +31,17 @@ public class ParseFixPatternWorker extends UntypedActor { private String patchesSourceCodeFilePath; private String editScriptSizesFilePath; private String buggyTokensFilePath; - private String alarmTypesFilePath; public ParseFixPatternWorker(String editScriptsFilePath, String patchesSourceCodeFilePath, - String buggyTokensFilePath, String editScriptSizesFilePath, String alarmTypesFilePath) { + String buggyTokensFilePath, String editScriptSizesFilePath) { this.editScriptsFilePath = editScriptsFilePath; this.patchesSourceCodeFilePath = patchesSourceCodeFilePath; this.editScriptSizesFilePath = editScriptSizesFilePath; this.buggyTokensFilePath = buggyTokensFilePath; - this.alarmTypesFilePath = alarmTypesFilePath; } public static Props props(final String editScriptsFile, final String patchesSourceCodeFile, final String buggyTokensFilePath, - final String editScriptSizesFilePath, final String alarmTypesFilePath) { + final String editScriptSizesFilePath) { return Props.create(new Creator() { private static final long serialVersionUID = -7615153844097275009L; @@ -51,7 +49,7 @@ public class ParseFixPatternWorker extends UntypedActor { @Override public ParseFixPatternWorker create() throws Exception { return new ParseFixPatternWorker(editScriptsFile, patchesSourceCodeFile, - buggyTokensFilePath, editScriptSizesFilePath, alarmTypesFilePath); + buggyTokensFilePath, editScriptSizesFilePath); } }); @@ -66,13 +64,11 @@ public class ParseFixPatternWorker extends UntypedActor { StringBuilder patchesSourceCode = new StringBuilder(); StringBuilder sizes = new StringBuilder(); StringBuilder tokens = new StringBuilder(); - StringBuilder alarmTypes = new StringBuilder(); StringBuilder testingInfo = new StringBuilder(); int id = msg.getId(); int counter = 0; - int testViolations = 0; int nullGumTreeResults = 0; int noSourceCodeChanges = 0; int noStatementChanges = 0; @@ -85,28 +81,15 @@ public class ParseFixPatternWorker extends UntypedActor { int timeouts = 0; StringBuilder builder = new StringBuilder(); - // Read violations with Null_Violation_Hunk or Illegal_Line_Position -// List uselessViolations = new ArrayList<>();// readUselessViolations("logs/FixedViolationCodeParseResults.log"); - for (MessageFile msgFile : files) { File revFile = msgFile.getRevFile(); File prevFile = msgFile.getPrevFile(); File diffentryFile = msgFile.getDiffEntryFile(); File positionFile = msgFile.getPositionFile(); - if (revFile.getName().toLowerCase().contains("test#") || revFile.getName().toLowerCase().contains("tests#")) { -// testViolations += countAlarms(positionFile, "#TestViolation:", uselessViolations); - testViolations += countAlarms(positionFile, "#TestViolation:"); + if (revFile.getName().toLowerCase().contains("test")) { continue; } -// Parser parser = null; -// if (containsAlarmTypes || positionFile != null) { -// parser = new FixedViolationHunkParser(positionFile); -// containsAlarmTypes = true; -// } else { -// parser = new CommitPatchSingleStatementParser(); -// } - FixedViolationHunkParser parser = new FixedViolationHunkParser(positionFile); -// parser.setUselessViolations(uselessViolations); + FixedViolationHunkParser parser = new FixedViolationHunkParser(); final ExecutorService executor = Executors.newSingleThreadExecutor(); // schedule the work @@ -128,20 +111,15 @@ public class ParseFixPatternWorker extends UntypedActor { if ("".equals(editScript)) { if (parser.resultType == 1) { nullGumTreeResults += countAlarms(positionFile, ""); -// System.err.println("#NullGumTreeResult:" + revFile.getName()); } else if (parser.resultType == 2) { noSourceCodeChanges += countAlarms(positionFile, ""); } else if (parser.resultType == 3) { noStatementChanges += countAlarms(positionFile, ""); -// } else if (parser.resultType == 4) { -// illegalV += countAlarms(positionFile, "", uselessViolations); } } else { editScripts.append(editScript); patchesSourceCode.append(parser.getPatchesSourceCode()); sizes.append(parser.getSizes()); -// tokens.append(parser.getTokensOfSourceCode()); - alarmTypes.append(parser.getAlarmTypes()); tokens.append(parser.getTokensOfSourceCode()); counter ++; @@ -154,26 +132,23 @@ public class ParseFixPatternWorker extends UntypedActor { patchesSourceCode.setLength(0); sizes.setLength(0); tokens.setLength(0); - FileHelper.outputToFile(alarmTypesFilePath + "alarmTypes_" + id + ".list", alarmTypes, true); - alarmTypes.setLength(0); - log.info("Worker #" + id +"Finish of parsing " + counter + " files..."); + log.info("Worker #" + id +" finialized parsing " + counter + " files..."); FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); testingInfo.setLength(0); } } } catch (TimeoutException e) { -// err.println("task timed out"); future.cancel(true); timeouts += countAlarms(positionFile, "#Timeout:"); -// System.err.println("#Timeout: " + revFile.getName()); + System.err.println("#Timeout: " + revFile.getName()); } catch (InterruptedException e) { timeouts += countAlarms(positionFile, "#TimeInterrupted:"); -// err.println("task interrupted"); -// System.err.println("#TimeInterrupted: " + revFile.getName()); + System.err.println("#TimeInterrupted: " + revFile.getName()); + e.printStackTrace(); } catch (ExecutionException e) { timeouts += countAlarms(positionFile, "#TimeAborted:"); -// err.println("task aborted"); -// System.err.println("#TimeAborted: " + revFile.getName()); + System.err.println("#TimeAborted: " + revFile.getName()); + e.printStackTrace(); } finally { executor.shutdownNow(); } @@ -189,58 +164,24 @@ public class ParseFixPatternWorker extends UntypedActor { sizes.setLength(0); tokens.setLength(0); - FileHelper.outputToFile(alarmTypesFilePath + "alarmTypes_" + id + ".list", alarmTypes, true); - alarmTypes.setLength(0); FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); testingInfo.setLength(0); } - String statistic = "TestViolations: " + testViolations + "\nNullGumTreeResults: " + nullGumTreeResults + "\nNoSourceCodeChanges: " + noSourceCodeChanges + + String statistic = "\nNullGumTreeResults: " + nullGumTreeResults + "\nNoSourceCodeChanges: " + noSourceCodeChanges + "\nNoStatementChanges: " + noStatementChanges + "\nNullDiffEntry: " + nullDiffEntry + "\nNullMatchedGumTreeResults: " + nullMappingGumTreeResults + "\nPureDeletion: " + pureDeletion + "\nLargeHunk: " + largeHunk + "\nNullSourceCode: " + nullSourceCode + "\nTestingInfo: " + testInfos + "\nTimeout: " + timeouts; FileHelper.outputToFile("OUTPUT/statistic_" + id + ".list", statistic, false); FileHelper.outputToFile("OUTPUT/UnfixedV_" + id + ".list", builder, false); - log.info("Worker #" + id +"Finish of parsing " + counter + " files..."); - log.info("Worker #" + id + " finished the work..."); + log.info("Worker #" + id +"finialized parsing " + counter + " files..."); + log.info("Worker #" + id + " finialized the work..."); this.getSender().tell("STOP", getSelf()); } else { unhandled(message); } } -// private List readUselessViolations(String filePath) { -// List uselessViolations = new ArrayList<>(); -// -// String content = FileHelper.readFile(filePath); -// BufferedReader reader = new BufferedReader(new StringReader(content)); -// String line = null; -// try { -// while ((line = reader.readLine()) != null) { -// if (line.startsWith("#")) { -// String[] elements = line.split(":"); -// String fileName = elements[1]; -// int startLine = Integer.parseInt(elements[2]); -// int endLine = Integer.parseInt(elements[3]); -// String violationType = elements[4]; -// -// Violation violation = new Violation(startLine, endLine, violationType); -// violation.setFileName(fileName); -// uselessViolations.add(violation); -// } -// } -// } catch (IOException e) { -// e.printStackTrace(); -// } finally { -// try { -// reader.close(); -// } catch (IOException e) { -// e.printStackTrace(); -// } -// } -// return uselessViolations; -// } - private int countAlarms(File positionFile, String type) {//, List uselessViolations) { int counter = 0; String content = FileHelper.readFile(positionFile); diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/SingleThreadParser.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/SingleThreadParser.java index 015cd6b..80b03db 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/SingleThreadParser.java +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/SingleThreadParser.java @@ -66,7 +66,7 @@ public class SingleThreadParser { testViolations += countAlarms(positionFile, "#TestViolation:"); continue; } - FixedViolationHunkParser parser = new FixedViolationHunkParser(positionFile); + FixedViolationHunkParser parser = new FixedViolationHunkParser(); // parser.setUselessViolations(uselessViolations); final ExecutorService executor = Executors.newSingleThreadExecutor(); diff --git a/src/main/java/edu/lu/uni/serval/config/Configuration.java b/src/main/java/edu/lu/uni/serval/config/Configuration.java index 90eb62a..b86ac0e 100644 --- a/src/main/java/edu/lu/uni/serval/config/Configuration.java +++ b/src/main/java/edu/lu/uni/serval/config/Configuration.java @@ -2,11 +2,11 @@ package edu.lu.uni.serval.config; public class Configuration { - public static final long SECONDS_TO_WAIT = 300000L; + public static final long SECONDS_TO_WAIT = 300L; - public static final String ROOT_PATH = "../FPM_Violations/"; // The root path of all output data. + public static String ROOT_PATH = "../FPM_Violations/"; // The root path of all output data. - public static final int HUNK_SIZE = 10; // The limitation of source code lines of each DiffEntry, which will be selected as training data. + public static int HUNK_SIZE = 10; // The limitation of source code lines of each DiffEntry, which will be selected as training data. public static final String BUGGY_TREE_SIGNAL = "BUGGY_TREE###"; // The starting signal of the tree of buggy source code . public static final String PATCH_SIGNAL = "PATCH###"; // The starting signal of each patch. @@ -86,9 +86,5 @@ public class Configuration { public static final String SUPERVISED_LEARNING_MODEL = ROOT_PATH + "TestingOutput/SupervisedLearningModel.zip"; public static final String FEATURES_OF_COMMON_CLUSTERS = ROOT_PATH + "FeaturesOfCommonClusters/"; - public static final String TESTING_DATA_BUGS90 = ROOT_PATH + "Bugs/bugs90.list"; - public static final String TESTING_DATA_BUGS80 = ROOT_PATH + "Bugs/bugs80.list"; - public static final String TESTING_DATA_BUGS70 = ROOT_PATH + "Bugs/bugs70.list"; - public static final String TESTING_DATA_BUGS60 = ROOT_PATH + "Bugs/bugs60.list"; } diff --git a/src/main/java/edu/lu/uni/serval/diffentry/DiffEntryHunk.java b/src/main/java/edu/lu/uni/serval/diffentry/DiffEntryHunk.java index 8c3c62e..ece32e5 100644 --- a/src/main/java/edu/lu/uni/serval/diffentry/DiffEntryHunk.java +++ b/src/main/java/edu/lu/uni/serval/diffentry/DiffEntryHunk.java @@ -1,5 +1,10 @@ package edu.lu.uni.serval.diffentry; +import java.util.ArrayList; +import java.util.List; + +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; + public class DiffEntryHunk { private int bugLineStartNum; @@ -10,6 +15,7 @@ public class DiffEntryHunk { private int buggyHunkSize; private int fixedHunkSize; private String file; + private List actionSets = new ArrayList<>(); public DiffEntryHunk(int bugLineStartNum, int fixLineStartNum, int bugRange, int fixRange) { super(); @@ -66,6 +72,10 @@ public class DiffEntryHunk { public void setFile(String file) { this.file = file; } + + public List getActionSets() { + return actionSets; + } @Override public String toString() { diff --git a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalActionSet.java b/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalActionSet.java index e304e87..7823826 100644 --- a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalActionSet.java +++ b/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalActionSet.java @@ -214,7 +214,7 @@ public class HierarchicalActionSet implements Comparable actionSet.toString(); List strList1 = actionSet.strList; for (String str1 : strList1) { - strList.add("----" + str1); + strList.add("---" + str1); } } } else { @@ -224,7 +224,7 @@ public class HierarchicalActionSet implements Comparable actionSet.toString(); List strList1 = actionSet.strList; for (String str1 : strList1) { - strList.add("----" + str1); + strList.add("---" + str1); } } } diff --git a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java b/src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java index 1b64b87..6dbe24f 100644 --- a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java +++ b/src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java @@ -361,237 +361,56 @@ public class HunkActionFilter { * @param prevFile * @return */ - public List filterActionsByModifiedRange2(List violations, + public List filterActionsByModifiedRange2(List diffentryHunks, List actionSets, File revFile, File prevFile) { - List selectedViolations = new ArrayList<>(); + List selectedViolations = new ArrayList<>(); CUCreator cuCreator = new CUCreator(); CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); if (prevUnit == null || revUnit == null) { - for (Violation violation : violations) { - this.unfixedViolations += "#NullMatchedGumTreeResult:" + revFile.getName() + ":" +violation.getStartLineNum() + ":" + - violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; - } return selectedViolations; } - for (Violation violation : violations) { - int violationStartLine = violation.getStartLineNum(); - int violationEndLine = violation.getEndLineNum(); - int bugHunkStartLine = violation.getBugStartLineNum(); + for (DiffEntryHunk diffentryHunk : diffentryHunks) { +// int violationEndLine = violationStartLine + diffentryHunk.getBugRange(); + int bugHunkStartLine = diffentryHunk.getBugLineStartNum(); + int bugHunkEndLine = bugHunkStartLine + diffentryHunk.getBugRange() - 1; + int fixHunkStartLine = diffentryHunk.getFixLineStartNum(); + int fixHunkEndLine = fixHunkStartLine + diffentryHunk.getFixedHunkSize() - 1; - if (!specialVioaltionTypes(violation, actionSets, prevUnit, revUnit)) { - if (bugHunkStartLine == 0) {// Null source code matched for this violation. -// String type = getType(violation); - } else if (bugHunkStartLine == -1) {// -// specialVioaltionTypes(violation, actionSets, prevUnit, revUnit); - } else { - int bugHunkEndLine = violation.getBugEndLineNum(); - int fixHunkStartLine = violation.getFixStartLineNum(); - int fixHunkEndLine = violation.getFixEndLineNum(); - int bugFixStartLineNum = violation.getBugFixStartLineNum(); - int bugFixEndLineNum = violation.getBugFixEndLineNum(); - - for (HierarchicalActionSet actionSet : actionSets) { - int actionBugStartLine = actionSet.getBugStartLineNum(); - if (actionBugStartLine == 0) { - actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); - } - int actionBugEndLine = actionSet.getBugEndLineNum(); - int actionFixStartLine = actionSet.getFixStartLineNum(); - int actionFixEndLine = actionSet.getFixEndLineNum(); - - String actionStr = actionSet.getActionString(); - if (actionStr.startsWith("INS")) { - if (fixHunkStartLine <= actionFixStartLine && actionFixEndLine <= fixHunkEndLine) { - if (actionBugStartLine != 0) { - if (violationStartLine <= actionBugEndLine && violationEndLine >= actionBugStartLine) { - violation.getActionSets().add(actionSet); - continue; - } - } - - // INS with MOV actions that are not identified in previous IF predicate, and pure INS actions - if (bugFixStartLineNum >= actionFixEndLine && actionFixStartLine <= bugFixEndLineNum && Math.abs(bugFixStartLineNum - actionFixStartLine) <= 3 - && Math.abs(bugFixEndLineNum - actionFixEndLine) <= 3) { - violation.getActionSets().add(actionSet); - } - } - } else {//if (!actionStr.startsWith("MOV")){// ignore move actions. - if (bugHunkStartLine <= actionBugStartLine && violationEndLine <= bugHunkEndLine) { - if (violationStartLine <= actionBugEndLine && violationEndLine >= actionBugStartLine) { - violation.getActionSets().add(actionSet); - } - } + for (HierarchicalActionSet actionSet : actionSets) { + int actionBugStartLine = actionSet.getBugStartLineNum(); + if (actionBugStartLine == 0) { + actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); + } + int actionBugEndLine = actionSet.getBugEndLineNum(); + int actionFixStartLine = actionSet.getFixStartLineNum(); + int actionFixEndLine = actionSet.getFixEndLineNum(); + + String actionStr = actionSet.getActionString(); + if (actionStr.startsWith("INS")) { + if (fixHunkStartLine <= actionFixEndLine && fixHunkEndLine >= actionFixStartLine ) { + if (actionBugStartLine != 0) { + diffentryHunk.getActionSets().add(actionSet); } } + } else { + if (bugHunkStartLine <= actionBugEndLine && bugHunkEndLine >= actionBugStartLine) { + diffentryHunk.getActionSets().add(actionSet); + } } } - if (violation.getActionSets().size() > 0) { - selectedViolations.add(violation); - } else { - this.unfixedViolations += "#NullMatchedGumTreeResult:" + revFile.getName() + ":" +violation.getStartLineNum() + ":" + - violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; + if (diffentryHunk.getActionSets().size() > 0) { + selectedViolations.add(diffentryHunk); } } return selectedViolations; } - public String unfixedViolations = ""; - private boolean specialVioaltionTypes(Violation violation, List actionSets, CompilationUnit prevUnit, CompilationUnit revUnit) { - String type = violation.getViolationType(); - - if ("NM_METHOD_NAMING_CONVENTION".equals(type) || "SE_NO_SUITABLE_CONSTRUCTOR_FOR_EXTERNALIZATION".equals(type)) { - int startLine = violation.getStartLineNum(); - for (HierarchicalActionSet actionSet : actionSets) { - int actionBugStartLine = actionSet.getBugStartLineNum(); - if (actionBugStartLine == 0) { - actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); - } - if (actionSet.getActionString().startsWith("UPD MethodDeclaration@@")) { - if (Math.abs(startLine - actionBugStartLine) <= 2) { - violation.getActionSets().add(actionSet); - break; - } - } - } - } else if ("EQ_DOESNT_OVERRIDE_EQUALS".equals(type)) { - for (HierarchicalActionSet actionSet : actionSets) { - int actionBugStartLine = actionSet.getBugStartLineNum(); - if (actionBugStartLine == 0) { - actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); - } - if (actionSet.getActionString().startsWith("INS MethodDeclaration@@MethodName:equals")) { - violation.getActionSets().add(actionSet); - break; - } - } - } else if ("HE_EQUALS_USE_HASHCODE".equals(type) || "HE_INHERITS_EQUALS_USE_HASHCODE".equals(type)) { - for (HierarchicalActionSet actionSet : actionSets) { - int actionBugStartLine = actionSet.getBugStartLineNum(); - if (actionBugStartLine == 0) { - actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); - } - if (actionSet.getActionString().startsWith("INS MethodDeclaration@@MethodName:hashCode")) { - violation.getActionSets().add(actionSet); - break; - } - } - } else if ("SE_NO_SUITABLE_CONSTRUCTOR".equals(type) || "RI_REDUNDANT_INTERFACES".equals(type)) { - int startLine = violation.getStartLineNum(); - for (HierarchicalActionSet actionSet : actionSets) { - int actionBugStartLine = actionSet.getBugStartLineNum(); - if (actionBugStartLine == 0) { - actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); - } - if (actionSet.getActionString().startsWith("UPD TypeDeclaration@@")) { - if (Math.abs(startLine - actionBugStartLine) <= 2) { - violation.getActionSets().add(actionSet); - break; - } - } - } - } else if ("CN_IDIOM".equals(type)) { // 202 23 - //add clone method. or update clone method - for (HierarchicalActionSet actionSet : actionSets) { - int actionBugStartLine = actionSet.getBugStartLineNum(); - if (actionBugStartLine == 0) { - actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); - } - if (actionSet.getActionString().startsWith("INS MethodDeclaration@@MethodName:clone")) { -// || actionSet.getActionString().startsWith("UPD MethodDeclaration@@clone")) { - violation.getActionSets().add(actionSet); - break; - } - } - } else if ("SE_NO_SERIALVERSIONID".equals(type) || "SE_COMPARATOR_SHOULD_BE_SERIALIZABLE".equals(type)) {// 12 1960 - // change superclass or interface, add field or remove @SuppressWarnings("serial"), some are inner class - for (HierarchicalActionSet actionSet : actionSets) { - int actionBugStartLine = actionSet.getBugStartLineNum(); - if (actionBugStartLine == 0) { - actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); - } - if (actionSet.getActionString().startsWith("INS FieldDeclaration@") && actionSet.getNode().getLabel().contains("serialVersionUID")) { - violation.getActionSets().add(actionSet); - break; - } - if (actionSet.getActionString().startsWith("UPD TypeDeclaration@")) { - int startLine = violation.getStartLineNum(); - if (Math.abs(startLine - actionBugStartLine) <= 2) { - violation.getActionSets().add(actionSet); - break; - } - } - } - } else { - return false; - } - return true; - } - - - private String getType(Violation violation) { - String type = violation.getViolationType(); - switch (type) { - case "CI_CONFUSED_INHERITANCE":// field - // update fieldDeclaration - break; - case "CO_ABSTRACT_SELF": - // java file is an interface, and delete compareTo(). - break; - case "EQ_ABSTRACT_SELF": - // java file is an interface, and delete equals(). - break; - case "SE_NO_SERIALVERSIONID": - // add a field: serialVersionUID - break; - case "EQ_COMPARETO_USE_OBJECT_EQUALS": - //Update or Delete compareTo(), Add equals() - break; - case "EQ_DOESNT_OVERRIDE_EQUALS": - //override equals() - break; - case "HE_SIGNATURE_DECLARES_HASHING_OF_UNHASHABLE_CLASS": - //remove equals() - break; - case "ME_MUTABLE_ENUM_FIELD": - // under enum, field add final keyword - break; - case "MF_CLASS_MASKS_FIELD": - // change super class or delete the field with a same name in super class. - break; - case "MS_SHOULD_BE_FINAL": - // add final to field - break; - case "STCAL_STATIC_SIMPLE_DATE_FORMAT_INSTANCE": - //remove public static final DateFormat DATE_FORMAT.... or SimpleDateFormat - break; - case "UUF_UNUSED_FIELD": - //remove unused fields. not sure - break; - case "UUF_UNUSED_PUBLIC_OR_PROTECTED_FIELD": - //remove unused fields. not sure - case "UWF_NULL_FIELD": - //update field, remove field - break; - case "UWF_UNWRITTEN_FIELD": - //field - break; - case "UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD": - //remove field - break; - case "VO_VOLATILE_REFERENCE_TO_ARRAY": - //field - break; - default: - break; - } - return null; - } /** * Filter out the modify actions, which are not in the DiffEntry hunks, without considering DiffEntry hunks.