diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java index 268b6e6..e0045fa 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java @@ -123,8 +123,8 @@ public class FixedViolationHunkParser extends FixedViolationParser { } if (fixStartLine == 0 && bugStartLine != 0) {// pure delete actions. - this.pureDeletions ++; - this.unfixedViolations += "#PureDeletion:" + revFile.getName() + ":" + violation.getStartLineNum() + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; +// this.pureDeletions ++; +// this.unfixedViolations += "#PureDeletion:" + revFile.getName() + ":" + violation.getStartLineNum() + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; // get the exact buggy code by violation's position. TODO later } @@ -137,9 +137,9 @@ public class FixedViolationHunkParser extends FixedViolationParser { // continue; } if ((bugEndLine - bugStartLine > Configuration.HUNK_SIZE && !isPureInsert) || fixEndLine - fixStartLine > Configuration.HUNK_SIZE) { - this.largeHunk ++; - this.unfixedViolations += "#LargeHunk:" + revFile.getName() + ":" + violation.getStartLineNum() - + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; +// this.largeHunk ++; +// this.unfixedViolations += "#LargeHunk:" + revFile.getName() + ":" + violation.getStartLineNum() +// + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; // continue; } @@ -151,9 +151,7 @@ public class FixedViolationHunkParser extends FixedViolationParser { this.nullSourceCode ++; this.unfixedViolations += "#NullSourceCode:" + revFile.getName() + ":" + violation.getStartLineNum() + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; -// continue; - patchSourceCode = getPatchSourceCode(prevFile, revFile, bugStartLine, bugEndLine, fixStartLine, fixEndLine, isPureInsert); - + continue; } } @@ -180,7 +178,7 @@ public class FixedViolationHunkParser extends FixedViolationParser { // continue; // } this.nullMappingGumTreeResult ++; - this.unfixedViolations += "#NullMatchedGumTreeResult:" + revFile.getName() + ":" +violation.getStartLineNum() + ":" + + this.unfixedViolations += "#NullMatchedGumTreeResult1:" + revFile.getName() + ":" +violation.getStartLineNum() + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; continue; } diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/SingleThreadParser.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/SingleThreadParser.java new file mode 100644 index 0000000..015cd6b --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/SingleThreadParser.java @@ -0,0 +1,190 @@ +package edu.lu.uni.serval.MultipleThreadsParser; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import edu.lu.uni.serval.FixPatternParser.RunnableParser; +import edu.lu.uni.serval.FixPatternParser.violations.FixedViolationHunkParser; +import edu.lu.uni.serval.FixPatternParser.violations.Violation; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.utils.FileHelper; + +public class SingleThreadParser { + + public static void main(String[] args) { + // output path + final String editScriptsFilePath = Configuration.EDITSCRIPTS_FILE_PATH; + final String patchesSourceCodeFilePath = Configuration.PATCH_SOURCECODE_FILE_PATH; + final String buggyTokensFilePath = Configuration.BUGGY_CODE_TOKEN_FILE_PATH; + final String editScriptSizesFilePath = Configuration.EDITSCRIPT_SIZES_FILE_PATH; + final String alarmTypesFilePath = Configuration.ALARM_TYPES_FILE_PATH; + FileHelper.deleteDirectory(editScriptsFilePath); + FileHelper.deleteDirectory(patchesSourceCodeFilePath); + FileHelper.deleteDirectory(buggyTokensFilePath); + FileHelper.deleteDirectory(editScriptSizesFilePath); + FileHelper.deleteDirectory(alarmTypesFilePath); + + final List files = AkkaParser.getMessageFiles(Configuration.GUM_TREE_INPUT); + + StringBuilder editScripts = new StringBuilder(); + StringBuilder patchesSourceCode = new StringBuilder(); + StringBuilder sizes = new StringBuilder(); + StringBuilder tokens = new StringBuilder(); + StringBuilder alarmTypes = new StringBuilder(); + StringBuilder testingInfo = new StringBuilder(); + + int counter = 0; + + int testViolations = 0; + int nullGumTreeResults = 0; + int noSourceCodeChanges = 0; + int noStatementChanges = 0; + int nullDiffEntry = 0; + int nullMappingGumTreeResults = 0; + int pureDeletion = 0; + int largeHunk = 0; + int nullSourceCode = 0; + int testInfos = 0; + int timeouts = 0; + StringBuilder builder = new StringBuilder(); + + for (MessageFile msgFile : files) { + File revFile = msgFile.getRevFile(); + File prevFile = msgFile.getPrevFile(); + File diffentryFile = msgFile.getDiffEntryFile(); + File positionFile = msgFile.getPositionFile(); + if (revFile.getName().toLowerCase().contains("test#") || revFile.getName().toLowerCase().contains("tests#")) { + testViolations += countAlarms(positionFile, "#TestViolation:"); + continue; + } + FixedViolationHunkParser parser = new FixedViolationHunkParser(positionFile); +// parser.setUselessViolations(uselessViolations); + + final ExecutorService executor = Executors.newSingleThreadExecutor(); + // schedule the work + final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser)); + try { + // wait for task to complete + future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); + + nullDiffEntry += parser.nullMatchedDiffEntry; + nullMappingGumTreeResults += parser.nullMappingGumTreeResult; + pureDeletion += parser.pureDeletions; + largeHunk += parser.largeHunk; + nullSourceCode += parser.nullSourceCode; + testInfos += parser.testInfos; + testingInfo.append(parser.testingInfo); + builder.append(parser.unfixedViolations); + + String editScript = parser.getAstEditScripts(); + if ("".equals(editScript)) { + if (parser.resultType == 1) { + nullGumTreeResults += countAlarms(positionFile, ""); + } else if (parser.resultType == 2) { + noSourceCodeChanges += countAlarms(positionFile, ""); + } else if (parser.resultType == 3) { + noStatementChanges += countAlarms(positionFile, ""); +// } else if (parser.resultType == 4) { + } + } else { + editScripts.append(editScript); + patchesSourceCode.append(parser.getPatchesSourceCode()); + sizes.append(parser.getSizes()); + alarmTypes.append(parser.getAlarmTypes()); + tokens.append(parser.getTokensOfSourceCode()); + + counter ++; + if (counter % 5000 == 0) { + FileHelper.outputToFile(editScriptsFilePath + "edistScripts.list", editScripts, true); + FileHelper.outputToFile(patchesSourceCodeFilePath + "patches.list", patchesSourceCode, true); + FileHelper.outputToFile(editScriptSizesFilePath + "sizes.list", sizes, true); + FileHelper.outputToFile(buggyTokensFilePath + "tokens.list", tokens, true); + editScripts.setLength(0); + patchesSourceCode.setLength(0); + sizes.setLength(0); + tokens.setLength(0); + FileHelper.outputToFile(alarmTypesFilePath + "alarmTypes.list", alarmTypes, true); + alarmTypes.setLength(0); + FileHelper.outputToFile("OUTPUT/testingInfo.list", testingInfo, true); + testingInfo.setLength(0); + } + } + } catch (TimeoutException e) { +// err.println("task timed out"); + future.cancel(true); + timeouts += countAlarms(positionFile, "#Timeout:"); +// System.err.println("#Timeout: " + revFile.getName()); + } catch (InterruptedException e) { + timeouts += countAlarms(positionFile, "#TimeInterrupted:"); +// err.println("task interrupted"); +// System.err.println("#TimeInterrupted: " + revFile.getName()); + } catch (ExecutionException e) { + timeouts += countAlarms(positionFile, "#TimeAborted:"); +// err.println("task aborted"); +// System.err.println("#TimeAborted: " + revFile.getName()); + } finally { + executor.shutdownNow(); + } + } + + if (sizes.length() > 0) { + FileHelper.outputToFile(editScriptsFilePath + "edistScripts.list", editScripts, true); + FileHelper.outputToFile(patchesSourceCodeFilePath + "patches.list", patchesSourceCode, true); + FileHelper.outputToFile(editScriptSizesFilePath + "sizes.list", sizes, true); + FileHelper.outputToFile(buggyTokensFilePath + "tokens.list", tokens, true); + editScripts.setLength(0); + patchesSourceCode.setLength(0); + sizes.setLength(0); + tokens.setLength(0); + + FileHelper.outputToFile(alarmTypesFilePath + "alarmTypes.list", alarmTypes, true); + alarmTypes.setLength(0); + FileHelper.outputToFile("OUTPUT/testingInfo.list", testingInfo, true); + testingInfo.setLength(0); + } + String statistic = "TestViolations: " + testViolations + "\nNullGumTreeResults: " + nullGumTreeResults + "\nNoSourceCodeChanges: " + noSourceCodeChanges + + "\nNoStatementChanges: " + noStatementChanges + "\nNullDiffEntry: " + nullDiffEntry + "\nNullMatchedGumTreeResults: " + nullMappingGumTreeResults + + "\nPureDeletion: " + pureDeletion + "\nLargeHunk: " + largeHunk + "\nNullSourceCode: " + nullSourceCode + + "\nTestingInfo: " + testInfos + "\nTimeout: " + timeouts; + FileHelper.outputToFile("OUTPUT/statistic.list", statistic, false); + FileHelper.outputToFile("OUTPUT/UnfixedV.list", builder, false); + } + + private static int countAlarms(File positionFile, String type) {//, List uselessViolations) { + int counter = 0; + String content = FileHelper.readFile(positionFile); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + try { + while ((line = reader.readLine()) != null) { + String[] elements = line.split(":"); + Violation v = new Violation(Integer.parseInt(elements[1]), Integer.parseInt(elements[2]), elements[0]); + String fileName = positionFile.getName().replace(".txt", ".java"); + v.setFileName(fileName); + counter ++; + if (!"".equals(type)) { + System.err.println(type + fileName + ":" + elements[1] + ":" + elements[2] + ":" + elements[0]); + } + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return counter; + } + +} diff --git a/src/main/java/edu/lu/uni/serval/statistics/Statistic.java b/src/main/java/edu/lu/uni/serval/statistics/Statistic.java index c415587..f1f1aae 100644 --- a/src/main/java/edu/lu/uni/serval/statistics/Statistic.java +++ b/src/main/java/edu/lu/uni/serval/statistics/Statistic.java @@ -71,18 +71,18 @@ public class Statistic { /** * Do statistics from two files: */ -// statistics("../FPM_Violations/RQ1/all-leafnodes-per-project-vtype.csv", "", map1, 16918530, 730); -//// statistics("../FPM_Violations/RQ1/distinct-fixed-summary-per-project-vtype.csv", "Fixed"); -// statistics("../FPM_Violations/RQ1/fixedViolations-v-1.0.csv", "Fixed_1.0", map2, 88927, 548); -//// fixedVSunfixed(); -// StringBuilder builder = new StringBuilder(); -// for (Map.Entry entry : map1.entrySet()) { -// String key = entry.getKey(); -// builder.append(key + "," + entry.getValue() + "," + (map2.containsKey(key) ? map2.get(key) : 0) + "\n"); -// } -// FileHelper.outputToFile(Configuration.ROOT_PATH + "RQ2/quantity-ratios.csv", builder, false); + statistics("../FPM_Violations/RQ1/all-leafnodes-per-project-vtype.csv", "", map1, 16918530, 730); +// statistics("../FPM_Violations/RQ1/distinct-fixed-summary-per-project-vtype.csv", "Fixed"); + statistics("../FPM_Violations/RQ1/fixedViolations-v-1.0.csv", "Fixed_1.0", map2, 88927, 548); +// fixedVSunfixed(); + StringBuilder builder = new StringBuilder(); + for (Map.Entry entry : map1.entrySet()) { + String key = entry.getKey(); + builder.append(key + "," + entry.getValue() + "," + (map2.containsKey(key) ? map2.get(key) : 0) + "\n"); + } + FileHelper.outputToFile(Configuration.ROOT_PATH + "RQ2/quantity-ratios.csv", builder, false); - statisticsOfFixedViolations(); +// statisticsOfFixedViolations(); //rsync -avP gaia-cluster:/work/users/kliu/FixPattern/FPM_Violations/UnfixedViolations/BC_UNCONFIRMED_CAST/Sizes.list Sizes/Sizes1.list // String s = "rsync -avP gaia-cluster:/work/users/kliu/FixPattern/FPM_Violations/UnfixedViolations_RQ3/"; @@ -120,7 +120,7 @@ public class Statistic { } public static void statisticsOfFixedViolations() { - String statistic = "../FPM_Violations/OUTPUT2/"; + String statistic = "../FPM_Violations/OUTPUT3/"; List files = FileHelper.getAllFiles(statistic, ".list"); int positions = 0; @@ -848,9 +848,9 @@ B: 4682 if (fixedQuantity == null) { builder.append(violationType + "," + 0 + "," + quantity + "," + quantity + ",0.0,1\n"); } else { - builder.append(violationType + "," + fixedQuantity + "," + (quantity - fixedQuantity) + builder.append(violationType + "," + fixedQuantity + "," + (quantity) + "," + quantity + "," + ((double)fixedQuantity) / ((double) quantity) - + "," + ((double)(quantity - fixedQuantity)) / ((double) quantity) + "\n"); + + "," + ((double)(quantity)) / ((double) quantity) + "\n"); } } FileHelper.outputToFile("../FPM_Violations/RQ1/Distribution-per-Fixed-type-VS-per-unFixed-type.csv", builder, false); @@ -893,7 +893,7 @@ B: 4682 String line= reader.readLine(); while ((line = reader.readLine()) != null) { String[] elements = line.split(","); - map.put(elements[0], Integer.valueOf(elements[1])); + map.put(elements[0], Integer.valueOf(elements[2])); } } catch (IOException e) { e.printStackTrace();