From c80627336a3d0dd2a72a1faebf93a34c8a255de9 Mon Sep 17 00:00:00 2001 From: Kui LIU Date: Wed, 16 Aug 2017 18:27:03 +0200 Subject: [PATCH] Add more information. --- .../violations/FixedViolationHunkParser.java | 50 ++++++++------- .../violations/FixedViolationParser.java | 10 ++- .../ParseFixPatternWorker.java | 23 +++++-- .../lu/uni/serval/config/Configuration.java | 2 +- .../gumtree/regroup/HunkActionFilter.java | 12 ++-- .../uni/serval/violation/parse/Statistic.java | 61 +++++++++++++++++++ 6 files changed, 122 insertions(+), 36 deletions(-) create mode 100644 src/main/java/edu/lu/uni/serval/violation/parse/Statistic.java diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java index ec764a5..0552abd 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java @@ -19,7 +19,6 @@ import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter; import edu.lu.uni.serval.gumtree.regroup.SimpleTree; import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; -import edu.lu.uni.serval.utils.ListSorter; /** * Parse fix violations with GumTree in terms of multiple statements. @@ -33,10 +32,13 @@ public class FixedViolationHunkParser extends FixedViolationParser { * ResultType: * 0: normal GumTree results. * 1: null GumTree result. + * 2: No source code changes. */ public int resultType = 0; public int nullMappingGumTreeResult = 0; public int pureDeletions = 0; + public int largeHunk = 0; + public int nullSourceCode = 0; public FixedViolationHunkParser() { } @@ -51,33 +53,36 @@ public class FixedViolationHunkParser extends FixedViolationParser { // TODO remove the modification of variable names or not? List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); // only remove non-statement source code, eg. method declaration - if (actionSets.size() > 0) { + if (actionSets == null) { + this.resultType = 1; + } else if (actionSets.size() == 0) { + this.resultType = 2; + } else { List violations = readPositionsAndAlarmTypes(); - if (violations.size() > 1) { - ListSorter sorter = new ListSorter<>(violations); - violations = sorter.sortAscending(); - } List diffentryHunks1 = new DiffEntryReader().readHunks2(diffentryFile); // Select hunks by positions of violations. for (Violation violation : violations) { - int startLineNum = violation.getStartLineNum(); - int endLineNum = violation.getEndLineNum(); + int violationStartLineNum = violation.getStartLineNum(); + int violationEndLineNum = violation.getEndLineNum(); for (int index = 0, hunkListSize = diffentryHunks1.size(); index < hunkListSize; index ++) { DiffEntryHunk hunk = diffentryHunks1.get(index); int startLine = hunk.getBugLineStartNum(); int range = hunk.getBugRange(); - if (startLineNum > startLine + range) continue; - if (endLineNum < startLine) break; + if (violationStartLineNum > startLine + range - 1) continue; + if (violationEndLineNum < startLine) break; if (violation.getBugStartLineNum() == 0) { violation.setBugStartLineNum(startLine); violation.setFixStartLineNum(hunk.getFixLineStartNum()); } - violation.setBugEndLineNum(startLine + range); - violation.setFixEndLineNum(hunk.getFixLineStartNum() + hunk.getFixRange()); + violation.setBugEndLineNum(startLine + range - 1); + violation.setFixEndLineNum(hunk.getFixLineStartNum() + hunk.getFixRange() - 1); violation.getHunks().add(hunk); } + if (violation.getBugStartLineNum() == 0 && violation.getBugEndLineNum() == 0) { + System.err.println("WRONG"); + } } //Filter out the modify actions, which are not in the DiffEntry hunks. @@ -195,7 +200,10 @@ public class FixedViolationHunkParser extends FixedViolationParser { isInsert = true; // continue; } - if (bugEndLine - bugStartLine >= Configuration.HUNK_SIZE || fixEndLine - fixStartLine >= Configuration.HUNK_SIZE) continue; + if (bugEndLine - bugStartLine > Configuration.HUNK_SIZE || fixEndLine - fixStartLine > Configuration.HUNK_SIZE) { + this.largeHunk ++; + continue; + } simpleTree.setChildren(children); simpleTree.setParent(null); @@ -203,7 +211,7 @@ public class FixedViolationHunkParser extends FixedViolationParser { // Source Code of patches. String patchSourceCode = getPatchSourceCode(prevFile, revFile, bugStartLine, bugEndLine, fixStartLine, fixEndLine, isInsert); if ("".equals(patchSourceCode)) { - this.nullMappingGumTreeResult ++; + this.nullSourceCode ++; continue; } @@ -218,22 +226,21 @@ public class FixedViolationHunkParser extends FixedViolationParser { // 4. semi-source code: String[] editScriptTokens = astEditScripts.split(" "); int size = editScriptTokens.length; - if (size == 1) { - this.nullMappingGumTreeResult ++; - continue; - } +// if (size == 1) { +// this.nullMappingGumTreeResult ++; +// continue; +// } String alarmType = violation.getAlarmType(); // String patchPosition = "\n" + revFile.getName() + "Position: " + violation.getStartLineNum() + " --> " + violation.getEndLineNum() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n"; String patchPosition = "\n" + "Position: " + violation.getStartLineNum() + " --> " + violation.getEndLineNum() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n"; String info = Configuration.PATCH_SIGNAL + "\nAlarm Type :" + violation.getAlarmType() + "\n" + patchPosition + patchSourceCode + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets, bugEndPosition, fixEndPosition) + "\n"; if (noUpdate(editScriptTokens)) { -// System.err.println(info); - this.testingInfo += info; if (!"SE_NO_SERIALVERSIONID".equals(alarmType)) { if (containsFiledDeclaration(hunkActionSets)) { this.nullMappingGumTreeResult ++; + this.testingInfo += info + revFile.getName() + "\n"; continue; } } @@ -252,6 +259,7 @@ public class FixedViolationHunkParser extends FixedViolationParser { } else { this.nullMappingGumTreeResult ++; + this.testingInfo += info + revFile.getName() + "\n"; continue; } } @@ -270,8 +278,6 @@ public class FixedViolationHunkParser extends FixedViolationParser { // this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n"; } - } else { - this.resultType = 1; } } diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java index ab84e2f..2ee0630 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java @@ -45,7 +45,11 @@ public class FixedViolationParser extends Parser { List actionSets = new ArrayList<>(); // GumTree results List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); - if (gumTreeResults != null && gumTreeResults.size() > 0) { + if (gumTreeResults == null) { + return null; + } else if (gumTreeResults.size() == 0){ + return actionSets; + } else { // Regroup GumTre results. List allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults); for (HierarchicalActionSet actionSet : allActionSets) { @@ -54,9 +58,9 @@ public class FixedViolationParser extends Parser { actionSets.add(actionSet); } } + + return actionSets; } - - return actionSets; } protected List readPositionsAndAlarmTypes() { diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java index cb45774..2ccd8ef 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java @@ -70,6 +70,7 @@ public class ParseFixPatternWorker extends UntypedActor { StringBuilder sizes = new StringBuilder(); StringBuilder tokens = new StringBuilder(); StringBuilder alarmTypes = new StringBuilder(); + StringBuilder testingInfo = new StringBuilder(); int id = msg.getId(); int counter = 0; @@ -77,8 +78,11 @@ public class ParseFixPatternWorker extends UntypedActor { int testAlarms = 0; int nullGumTreeResults = 0; int nullMappingGumTreeResults = 0; + int noSourceCodeChagnes = 0; int pureDeletion = 0; int expNums = 0; + int largeHunk = 0; + int nullSourceCode = 0; for (MessageFile msgFile : files) { File revFile = msgFile.getRevFile(); @@ -107,11 +111,16 @@ public class ParseFixPatternWorker extends UntypedActor { nullMappingGumTreeResults += parser.nullMappingGumTreeResult; pureDeletion += parser.pureDeletions; + largeHunk += parser.largeHunk; + nullSourceCode += parser.nullSourceCode; + testingInfo.append(parser.testingInfo); String editScript = parser.getAstEditScripts(); if ("".equals(editScript)) { if (parser.resultType == 1) { nullGumTreeResults += countAlarms(positionFile); + } else if (parser.resultType == 2) { + noSourceCodeChagnes += countAlarms(positionFile); } } else { editScripts.append(editScript); @@ -134,17 +143,19 @@ public class ParseFixPatternWorker extends UntypedActor { FileHelper.outputToFile(alarmTypesFilePath + "alarmTypes_" + id + ".list", alarmTypes, true); alarmTypes.setLength(0); log.info("Worker #" + id +"Finish of parsing " + counter + " files..."); + FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); + testingInfo.setLength(0); } } } catch (TimeoutException e) { err.println("task timed out"); future.cancel(true); - expNums ++; + expNums += countAlarms(positionFile); } catch (InterruptedException e) { - expNums ++; + expNums += countAlarms(positionFile); err.println("task interrupted"); } catch (ExecutionException e) { - expNums ++; + expNums += countAlarms(positionFile); err.println("task aborted"); } finally { executor.shutdownNow(); @@ -163,8 +174,12 @@ public class ParseFixPatternWorker extends UntypedActor { FileHelper.outputToFile(alarmTypesFilePath + "alarmTypes_" + id + ".list", alarmTypes, true); alarmTypes.setLength(0); + FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); + testingInfo.setLength(0); } - String statistic = "testAlarms: " + testAlarms + "\nnullGumTreeResults: " + nullGumTreeResults + "\nnullMappingGumTreeResults: " + nullMappingGumTreeResults + "\npureDeletion: " + pureDeletion + "\nTimeout: " + expNums; + String statistic = "testAlarms: " + testAlarms + "\nnullGumTreeResults: " + nullGumTreeResults + "\nnullMappingGumTreeResults: " + nullMappingGumTreeResults + + "\nnoSourceCodeChagnes: " + noSourceCodeChagnes + "\npureDeletion: " + pureDeletion + "\nTimeout: " + expNums + + "\nlargeHunk: " + largeHunk + "\nnullSourceCode: " + nullSourceCode; FileHelper.outputToFile("OUTPUT/statistic_" + id + ".list", statistic, false); log.info("Worker #" + id +"Finish of parsing " + counter + " files..."); diff --git a/src/main/java/edu/lu/uni/serval/config/Configuration.java b/src/main/java/edu/lu/uni/serval/config/Configuration.java index 57a041a..bd05254 100644 --- a/src/main/java/edu/lu/uni/serval/config/Configuration.java +++ b/src/main/java/edu/lu/uni/serval/config/Configuration.java @@ -6,7 +6,7 @@ public class Configuration { private static final String ROOT_PATH = "../FPM_Violations/"; // The root path of all output data. - public static final int HUNK_SIZE = 7; // The limitation of source code lines of each DiffEntry, which will be selected as training data. + public static final int HUNK_SIZE = 10; // The limitation of source code lines of each DiffEntry, which will be selected as training data. public static final String BUGGY_TREE_SIGNAL = "BUGGY_TREE###"; // The starting signal of the tree of buggy source code . public static final String PATCH_SIGNAL = "PATCH###"; // The starting signal of each patch. diff --git a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java b/src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java index b117a55..512b461 100644 --- a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java +++ b/src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java @@ -393,11 +393,11 @@ public class HunkActionFilter { violation.getActionSets().add(actionSet); } } else { - if (isRanged(actionSet, violation)) { - if (Math.abs(violationStartLine - actionFixStartLine) <= 5 - && Math.abs(violationEndLine - actionFixEndLine) <= 5) { - violation.getActionSets().add(actionSet); - } +// if (isRanged(actionSet, violation)) { +// } + if (Math.abs(violationStartLine - actionFixStartLine) <= 5 + && Math.abs(violationEndLine - actionFixEndLine) <= 5) { + violation.getActionSets().add(actionSet); } } } @@ -432,7 +432,7 @@ public class HunkActionFilter { int bugEndLine = bugStartLine + hunk.getBugRange(); int fixStartLine = hunk.getFixLineStartNum(); int fixEndLine = fixStartLine + hunk.getFixRange(); - if (fixStartLine > actionEndLine || bugStartLine > violationEndLine) break; +// if (fixStartLine > actionEndLine || bugStartLine > violationEndLine) break; if (fixEndLine < actionStartLine || bugEndLine < violationStartLine) continue; String hunkContent = hunk.getHunk(); diff --git a/src/main/java/edu/lu/uni/serval/violation/parse/Statistic.java b/src/main/java/edu/lu/uni/serval/violation/parse/Statistic.java new file mode 100644 index 0000000..6e14489 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/violation/parse/Statistic.java @@ -0,0 +1,61 @@ +package edu.lu.uni.serval.violation.parse; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.List; + +import edu.lu.uni.serval.utils.FileHelper; + +public class Statistic { + public static void main(String[] args) { + String statistic = "../FPM_Violations/OUTPUT"; + List files = FileHelper.getAllFiles(statistic, ".list"); + int testAlarms = 0; + int nullGumTreeResults = 0; + int nullMappingGumTreeResults = 0; + int pureDeletion = 0; + int timeout = 0; + for (File file : files) { + String content = FileHelper.readFile(file); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + try { + while ((line = reader.readLine()) != null) { + if (line.startsWith("test")) { + testAlarms += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("nullGum")) { + nullGumTreeResults += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("nullMap")) { + nullMappingGumTreeResults += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("pure")) { + pureDeletion += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("Time")) { + timeout += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + +// TestAlarms: 5175 +// nullGumTreeResults: 13449 +// nullMappingGumTreeResults: 33010 +// pureDeletion: 7598 +// Timeout: 263 + + System.out.println("TestAlarms: " + testAlarms); + System.out.println("nullGumTreeResults: " + nullGumTreeResults); + System.out.println("nullMappingGumTreeResults: " + nullMappingGumTreeResults); + System.out.println("pureDeletion: " + pureDeletion); + System.out.println("Timeout: " + timeout); + } +}