From 183f63608aef344cf415910a97932e8c610a48b6 Mon Sep 17 00:00:00 2001 From: Kui LIU Date: Thu, 21 Sep 2017 13:42:01 +0200 Subject: [PATCH] Upsdate --- .gitignore | 1 + .../violations/FixedViolationHunkParser.java | 180 +++++++---- .../ParseFixPatternWorker.java | 5 +- .../MultipleThreadsParser/WorkMessage.java | 2 +- .../MultipleThreadsParser2/AkkaMatcher.java | 1 + .../MultipleThreadsParser2/MessageFile.java | 39 --- .../MultipleThreadsParser3/AkkaParser.java | 86 ++++++ .../ParseFixPatternActor.java | 80 +++++ .../ParseFixPatternWorker.java | 96 ++++++ .../lu/uni/serval/config/Configuration.java | 2 +- .../regroup/HierarchicalRegrouper.java | 12 +- .../gumtree/regroup/HunkActionFilter.java | 149 +++------ .../serval/gumtree/regroup/SimplifyTree.java | 42 ++- .../lu/uni/serval/statistics/Statistic.java | 243 +++++++++------ .../serval/violation/code/parser/Test.java | 19 +- .../code/parser/ViolatePositionReader.java | 115 +++++++ .../code/parser/ViolationCodeParser.java | 114 +++++++ .../code/parser/ViolationSourceCodeTree.java | 285 +++++++++++++++++- .../violation/parse/TestViolationParser.java | 4 +- 19 files changed, 1137 insertions(+), 338 deletions(-) delete mode 100644 src/main/java/edu/lu/uni/serval/MultipleThreadsParser2/MessageFile.java create mode 100644 src/main/java/edu/lu/uni/serval/MultipleThreadsParser3/AkkaParser.java create mode 100644 src/main/java/edu/lu/uni/serval/MultipleThreadsParser3/ParseFixPatternActor.java create mode 100644 src/main/java/edu/lu/uni/serval/MultipleThreadsParser3/ParseFixPatternWorker.java create mode 100644 src/main/java/edu/lu/uni/serval/violation/code/parser/ViolatePositionReader.java diff --git a/.gitignore b/.gitignore index 23c6385..e82d5e7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ OUTPUT/ *.list *.csv *.pdf +logs/ GumTreeInput/ GumTreeInput1/ GumTreeOutput1/ diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java index ad3260a..6cf9d4d 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java @@ -46,6 +46,8 @@ public class FixedViolationHunkParser extends FixedViolationParser { setPositionFile(positionFile); } + public String unfixedViolations = ""; + @Override public void parseFixPatterns(File prevFile, File revFile, File diffentryFile) { List allViolations = readViolations(prevFile, revFile); @@ -61,20 +63,24 @@ public class FixedViolationHunkParser extends FixedViolationParser { type = "#NoStatementChange:"; } for (Violation v : allViolations) { + unfixedViolations += type + revFile.getName() + ":" + v.getStartLineNum() + ":" + v.getEndLineNum() + ":" + v.getViolationType() + "\n"; System.err.println(type + revFile.getName() + ":" + v.getStartLineNum() + ":" + v.getEndLineNum() + ":" + v.getViolationType()); } } else { List diffentryHunks = new DiffEntryReader().readHunks2(diffentryFile); // Identify DiffEntry hunks by positions of violations. List violations = identifyFixRangeHeuristically(allViolations, diffentryHunks, revFile); - + //Filter out the modify actions, which are not in the DiffEntry hunks. HunkActionFilter hunkFilter = new HunkActionFilter(); List selectedViolations = hunkFilter.filterActionsByModifiedRange2(violations, actionSets, revFile, prevFile); + this.unfixedViolations += hunkFilter.unfixedViolations; this.nullMappingGumTreeResult += violations.size() - selectedViolations.size(); for (Violation violation : selectedViolations) { - List hunkActionSets = violation.getActionSets(); + List hunkActionSets = violation.getActionSets(); + // multiple UPD, and some UPD contain other UPD. + removeOverlapperdUPD(hunkActionSets); // // Remove overlapped UPD and INS, MOV // List addActions = new ArrayList<>(); // List insertActions = new ArrayList<>(); @@ -113,26 +119,7 @@ public class FixedViolationHunkParser extends FixedViolationParser { int fixEndLine = 0; int bugEndPosition = 0; int fixEndPosition = 0; - /* - * Convert the ITree of buggy code to a simple tree. - * It will be used to compute the similarity. - */ - SimpleTree simpleTree = new SimpleTree(); - simpleTree.setLabel("Block"); - simpleTree.setNodeType("Block"); - List children = new ArrayList<>(); - String astEditScripts = ""; for (HierarchicalActionSet hunkActionSet : hunkActionSets) { -// /** -// * Select edit scripts for deep learning. -// * Edit scripts will be used to mine common fix patterns. -// */ -// // 1. First level: AST node type. -// astEditScripts += getASTEditScripts(hunkActionSet); -// // 2. source code: raw tokens -// // 3. abstract identifiers: -// // 4. semi-source code: - int actionBugStart = hunkActionSet.getBugStartLineNum(); int actionBugEnd = hunkActionSet.getBugEndLineNum(); int actionFixStart = hunkActionSet.getFixStartLineNum(); @@ -157,16 +144,40 @@ public class FixedViolationHunkParser extends FixedViolationParser { } } + if (fixStartLine == 0 && bugStartLine == 0) { + this.unfixedViolations += "#WRONG: " + revFile.getName() + ":" + violation.getStartLineNum() + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; + continue; + } + if (fixStartLine == 0 && bugStartLine != 0) { // pure delete actions. this.pureDeletions ++; + this.unfixedViolations += "#PureDeletion:" + revFile.getName() + ":" + violation.getStartLineNum() + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; System.err.println("#PureDeletion:" + revFile.getName() + ":" + violation.getStartLineNum() + ":" + violation.getEndLineNum() + ":" + violation.getViolationType()); // get the exact buggy code by violation's position. TODO later } + /* + * Convert the ITree of buggy code to a simple tree. + * It will be used to compute the similarity. + */ + SimpleTree simpleTree = new SimpleTree(); + simpleTree.setLabel("Block"); + simpleTree.setNodeType("Block"); + List children = new ArrayList<>(); + String astEditScripts = ""; for (HierarchicalActionSet hunkActionSet : hunkActionSets) { // TODO simplify buggy tree with buggy code. + /** +// * Select edit scripts for deep learning. +// * Edit scripts will be used to mine common fix patterns. +// */ +// // 1. First level: AST node type. +// astEditScripts += getASTEditScripts(hunkActionSet); +// // 2. source code: raw tokens +// // 3. abstract identifiers: +// // 4. semi-source code: SimplifyTree abstractIdentifier = new SimplifyTree(); abstractIdentifier.abstractTree(hunkActionSet, bugEndPosition); SimpleTree simpleT = hunkActionSet.getSimpleTree(); @@ -178,14 +189,16 @@ public class FixedViolationHunkParser extends FixedViolationParser { // if (children.size() == 0) continue; boolean isPureInsert = false; - if (bugStartLine == 0) { + if (bugStartLine == 0 && violation.getBugStartLineNum() > 0) { bugStartLine = violation.getStartLineNum(); bugEndLine = violation.getEndLineNum(); isPureInsert = true; // continue; } - if (bugEndLine - bugStartLine > Configuration.HUNK_SIZE || fixEndLine - fixStartLine > Configuration.HUNK_SIZE) { + if ((bugEndLine - bugStartLine > Configuration.HUNK_SIZE && !isPureInsert) || fixEndLine - fixStartLine > Configuration.HUNK_SIZE) { this.largeHunk ++; + this.unfixedViolations += "#LargeHunk:" + revFile.getName() + ":" + violation.getStartLineNum() + + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; System.err.println("#LargeHunk:" + revFile.getName() + ":" + violation.getStartLineNum() + ":" + violation.getEndLineNum() + ":" + violation.getViolationType()); continue; @@ -198,6 +211,8 @@ public class FixedViolationHunkParser extends FixedViolationParser { String patchSourceCode = getPatchSourceCode(prevFile, revFile, bugStartLine, bugEndLine, fixStartLine, fixEndLine, isPureInsert); if ("".equals(patchSourceCode)) { this.nullSourceCode ++; + this.unfixedViolations += "#NullSourceCode:" + revFile.getName() + ":" + violation.getStartLineNum() + + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; System.err.println("#NullSourceCode:" + revFile.getName() + ":" + violation.getStartLineNum() + ":" + violation.getEndLineNum() + ":" + violation.getViolationType()); continue; @@ -223,46 +238,50 @@ public class FixedViolationHunkParser extends FixedViolationParser { // String patchPosition = "\n" + revFile.getName() + "Position: " + violation.getStartLineNum() + " --> " + violation.getEndLineNum() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n"; // String patchPosition = "\n" + "Position: " + violation.getStartLineNum() + " --> " + violation.getEndLineNum() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n"; // String info = Configuration.PATCH_SIGNAL + "\nAlarm Type :" + violation.getAlarmType() + "\n" + patchPosition + patchSourceCode + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets, bugEndPosition, fixEndPosition) + "\n"; - String patchPosition = "\n" + "Position: " + violation.getStartLineNum() + " --> " + violation.getEndLineNum() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n"; - String info = Configuration.PATCH_SIGNAL + "\nAlarm Type :" + violation.getViolationType() + "\n" + patchSourceCode + "\n" + patchPosition + revFile.getName() + "\n"; - if (noUpdate(editScriptTokens)) { - - if (!"SE_NO_SERIALVERSIONID".equals(alarmType)) { - if (containsFiledDeclaration(hunkActionSets)) { -// this.nullMappingGumTreeResult ++; //TODO - this.testingInfo += info + revFile.getName() + "\n"; - this.testingInfo += "#TestingInfo: " + revFile.getName() + ":" + violation.getStartLineNum() - + ":" + violation.getEndLineNum() + ":" + violation.getViolationType(); - System.err.println("#TestingInfo:" + revFile.getName() + ":" + violation.getStartLineNum() - + ":" + violation.getEndLineNum() + ":" + violation.getViolationType()); - this.testInfos ++; - continue; - } - } - - if ("UL_UNRELEASED_LOCK".equals(alarmType) || - "SF_SWITCH_NO_DEFAULT".equals(alarmType) || - "SF_SWITCH_FALLTHROUGH".equals(alarmType) || - "SE_NO_SERIALVERSIONID".equals(alarmType) || - "REC_CATCH_EXCEPTION".equals(alarmType) || - "OS_OPEN_STREAM".equals(alarmType) || - "NP_ALWAYS_NULL".equals(alarmType) || - "IS2_INCONSISTENT_SYNC".equals(alarmType) || - "EI_EXPOSE_REP".equals(alarmType) ) { - - } else if (containSpecialStmt(hunkActionSets, bugEndPosition, fixEndPosition)) { - - } else { -// this.nullMappingGumTreeResult ++; // TODO - this.testingInfo += info + revFile.getName() + "\n"; - this.testingInfo += "#TestingInfo: " + revFile.getName() + ":" + violation.getStartLineNum() - + ":" + violation.getEndLineNum() + ":" + violation.getViolationType(); - System.err.println("#TestingInfo:" + revFile.getName() + ":" + violation.getStartLineNum() - + ":" + violation.getEndLineNum() + ":" + violation.getViolationType()); - this.testInfos ++; - continue; - } - } +// String patchPosition = "\n" + "Position: " + violation.getStartLineNum() + " --> " + violation.getEndLineNum() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n"; + String info = Configuration.PATCH_SIGNAL + "\nAlarm Type :" + violation.getViolationType() + "\n" + patchSourceCode + "\n\n";// + patchPosition + revFile.getName() + "\n"; +// if (noUpdate(editScriptTokens)) { +// +// if (!"SE_NO_SERIALVERSIONID".equals(alarmType)) { +// if (containsFiledDeclaration(hunkActionSets)) { +//// this.nullMappingGumTreeResult ++; //TODO +// this.testingInfo += info + revFile.getName() + "\n"; +// this.testingInfo += "#TestingInfo: " + revFile.getName() + ":" + violation.getStartLineNum() +// + ":" + violation.getEndLineNum() + ":" + violation.getViolationType(); +// this.unfixedViolations += "#TestingInfo:" + revFile.getName() + ":" + violation.getStartLineNum() +// + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; +// System.err.println("#TestingInfo:" + revFile.getName() + ":" + violation.getStartLineNum() +// + ":" + violation.getEndLineNum() + ":" + violation.getViolationType()); +// this.testInfos ++; +// continue; +// } +// } +// +// if ("UL_UNRELEASED_LOCK".equals(alarmType) || +// "SF_SWITCH_NO_DEFAULT".equals(alarmType) || +// "SF_SWITCH_FALLTHROUGH".equals(alarmType) || +// "SE_NO_SERIALVERSIONID".equals(alarmType) || +// "REC_CATCH_EXCEPTION".equals(alarmType) || +// "OS_OPEN_STREAM".equals(alarmType) || +// "NP_ALWAYS_NULL".equals(alarmType) || +// "IS2_INCONSISTENT_SYNC".equals(alarmType) || +// "EI_EXPOSE_REP".equals(alarmType) ) { +// +// } else if (containSpecialStmt(hunkActionSets, bugEndPosition, fixEndPosition)) { +// +// } else { +//// this.nullMappingGumTreeResult ++; // TODO +// this.testingInfo += info + revFile.getName() + "\n"; +// this.testingInfo += "#TestingInfo: " + revFile.getName() + ":" + violation.getStartLineNum() +// + ":" + violation.getEndLineNum() + ":" + violation.getViolationType(); +// this.unfixedViolations += "#TestingInfo:" + revFile.getName() + ":" + violation.getStartLineNum() +// + ":" + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; +// System.err.println("#TestingInfo:" + revFile.getName() + ":" + violation.getStartLineNum() +// + ":" + violation.getEndLineNum() + ":" + violation.getViolationType()); +// this.testInfos ++; +// continue; +// } +// } this.patchesSourceCode += info; this.sizes += size + "\n"; @@ -281,6 +300,32 @@ public class FixedViolationHunkParser extends FixedViolationParser { } } + private void removeOverlapperdUPD(List actionSets) { + if (actionSets.size() == 1) { + return; + } + List updates = new ArrayList<>(); + for (HierarchicalActionSet actionSet : actionSets) { + if (actionSet.getActionString().startsWith("UPD")) { + updates.add(actionSet); + } + } + + List overlappedUpdates = new ArrayList<>(); + if (updates.size() > 1) { + for (HierarchicalActionSet update : updates) { + for (HierarchicalActionSet update2 : updates) { + if (update.equals(update2)) continue; + if (update.getBugStartLineNum() <= update2.getBugStartLineNum() + && update2.getBugEndLineNum() <= update.getBugEndLineNum()) { + overlappedUpdates.add(update); + } + } + } + } + actionSets.removeAll(overlappedUpdates); + } + // private List getExactActionSets(Violation violation, // List hunkActionSets) { // List deleteActionSets = new ArrayList<>(); @@ -332,13 +377,12 @@ public class FixedViolationHunkParser extends FixedViolationParser { (violationStartLineNum <= bugS && bugS <= violationEndLineNum)) { if (fixS == 0) fixS = currentFixLine; if (bugFixStartLine == -1) bugFixStartLine = currentFixLine; - fixE = currentFixLine; if (isFixRange) bugFixEndLine = currentFixLine; + fixE = currentFixLine; } else { fixedLines = 0; bugS = 0; } -// bugS = 0; // fixS = 0; } else if (line.startsWith("-")) { currentBugLine ++; if (bugS == 0 || fixS != 0) bugS = currentBugLine;// currentBugLine may be larger than the violation end line. @@ -414,10 +458,14 @@ public class FixedViolationHunkParser extends FixedViolationParser { if (violation.getHunks().size() == 0) { // This fixed violation cannot be matched with a DiffEntry, it is difficult to identify related source code change for it. this.nullMatchedDiffEntry ++; + this.unfixedViolations += "#NullDiffEntry: " + revFile.getName() + " : " +violation.getStartLineNum() + " : " + + violation.getEndLineNum() + " : " + violation.getViolationType() + "\n"; System.err.println("#NullDiffEntry: " + revFile.getName() + " : " +violation.getStartLineNum() + " : " + violation.getEndLineNum() + " : " + violation.getViolationType()); } else { selectedViolations.add(violation); } + } else { + selectedViolations.add(violation); } } @@ -439,11 +487,9 @@ public class FixedViolationHunkParser extends FixedViolationParser { if (hunkActionSet.getAstNodeType().equals("IfStatement")) { return true; } - if (hunkActionSet.getAstNodeType().equals("ThrowStatement")) { return true; } - if (hunkActionSet.getAstNodeType().equals("ReturnStatement")) { return true; } diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java index 2bd9115..2e2d5ae 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java @@ -83,6 +83,7 @@ public class ParseFixPatternWorker extends UntypedActor { int nullSourceCode = 0; int testInfos = 0; int timeouts = 0; + StringBuilder builder = new StringBuilder(); // Read violations with Null_Violation_Hunk or Illegal_Line_Position // List uselessViolations = new ArrayList<>();// readUselessViolations("logs/FixedViolationCodeParseResults.log"); @@ -121,6 +122,7 @@ public class ParseFixPatternWorker extends UntypedActor { nullSourceCode += parser.nullSourceCode; testInfos += parser.testInfos; testingInfo.append(parser.testingInfo); + builder.append(parser.unfixedViolations); String editScript = parser.getAstEditScripts(); if ("".equals(editScript)) { @@ -195,8 +197,9 @@ public class ParseFixPatternWorker extends UntypedActor { String statistic = "TestViolations: " + testViolations + "\nNullGumTreeResults: " + nullGumTreeResults + "\nNoSourceCodeChanges: " + noSourceCodeChanges + "\nNoStatementChanges: " + noStatementChanges + "\nNullDiffEntry: " + nullDiffEntry + "\nNullMatchedGumTreeResults: " + nullMappingGumTreeResults + "\nPureDeletion: " + pureDeletion + "\nLargeHunk: " + largeHunk + "\nNullSourceCode: " + nullSourceCode + - "\nTestingInfo: " + testInfos + "\nTimeout: " + timeouts;// + "\nIllegalV: " + illegalV; + "\nTestingInfo: " + testInfos + "\nTimeout: " + timeouts; FileHelper.outputToFile("OUTPUT/statistic_" + id + ".list", statistic, false); + FileHelper.outputToFile("OUTPUT/UnfixedV_" + id + ".list", builder, false); log.info("Worker #" + id +"Finish of parsing " + counter + " files..."); log.info("Worker #" + id + " finished the work..."); diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/WorkMessage.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/WorkMessage.java index e442f55..9d30eb6 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/WorkMessage.java +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/WorkMessage.java @@ -21,4 +21,4 @@ public class WorkMessage { return msgFiles; } -} +} \ No newline at end of file diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser2/AkkaMatcher.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser2/AkkaMatcher.java index e02b90b..0f59221 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser2/AkkaMatcher.java +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser2/AkkaMatcher.java @@ -15,6 +15,7 @@ import org.slf4j.LoggerFactory; import akka.actor.ActorRef; import akka.actor.ActorSystem; +import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; import edu.lu.uni.serval.config.Configuration; import edu.lu.uni.serval.utils.FileHelper; import edu.lu.uni.serval.utils.MapSorter; diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser2/MessageFile.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser2/MessageFile.java deleted file mode 100644 index 7cd6ff2..0000000 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser2/MessageFile.java +++ /dev/null @@ -1,39 +0,0 @@ -package edu.lu.uni.serval.MultipleThreadsParser2; - -import java.io.File; - -public class MessageFile { - - private File revFile; - private File prevFile; - private File diffEntryFile; - private File positionFile; - - public MessageFile(File revFile, File prevFile, File diffEntryFile) { - super(); - this.revFile = revFile; - this.prevFile = prevFile; - this.diffEntryFile = diffEntryFile; - } - - public File getRevFile() { - return revFile; - } - - public File getPrevFile() { - return prevFile; - } - - public File getDiffEntryFile() { - return diffEntryFile; - } - - public File getPositionFile() { - return positionFile; - } - - public void setPositionFile(File positionFile) { - this.positionFile = positionFile; - } - -} diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser3/AkkaParser.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser3/AkkaParser.java new file mode 100644 index 0000000..7f04dea --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser3/AkkaParser.java @@ -0,0 +1,86 @@ +package edu.lu.uni.serval.MultipleThreadsParser3; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; +import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.utils.FileHelper; + +/** + * Multi-thread parser of parsing source code of unfixed violations. + * + * @author kui.liu + * + */ +public class AkkaParser { + + private static Logger log = LoggerFactory.getLogger(AkkaParser.class); + + @SuppressWarnings("deprecation") + public static void main(String[] args) { + String violationType = Configuration.ROOT_PATH + "UnfixedViolations/"; + File file = new File(violationType); + File[] violationTypes = file.listFiles(); + for (File violationT : violationTypes) { + if (violationT.isDirectory()) { + violationType = violationT.getName(); + // input data + log.info("Get the input data..."); + final List msgFiles = getMessageFiles(Configuration.GUM_TREE_INPUT, violationType); + log.info("MessageFiles: " + msgFiles.size()); + + // output path + final String sourceCodeFilesPath = Configuration.ROOT_PATH + "UnfixedViolations/" + violationType + "/"; + FileHelper.deleteDirectory(sourceCodeFilesPath); + + ActorSystem system = null; + ActorRef parsingActor = null; + int numberOfWorkers = 200; + final WorkMessage msg = new WorkMessage(0, msgFiles); + try { + log.info("Akka begins..."); + system = ActorSystem.create("Mining-Pattern-System-" + violationType); + parsingActor = system.actorOf(ParseFixPatternActor.props(numberOfWorkers, sourceCodeFilesPath, violationType), "mine-pattern-actor-" + violationType); + parsingActor.tell(msg, ActorRef.noSender()); + } catch (Exception e) { + system.shutdown(); + e.printStackTrace(); + } + } + } + } + + + /** + * Get violation-related files. + * + * @param gumTreeInput + * @return + */ + public static List getMessageFiles(String inputPath, String violationType) { + File sourceCodeFilesPath = new File(inputPath + "UnfixedViolations/" + violationType + "/"); + File[] sourceCodeFiles = sourceCodeFilesPath.listFiles(); // project folders + List msgFiles = new ArrayList<>(); + + for (File sourceCodeFile : sourceCodeFiles) { + if (sourceCodeFile.getName().endsWith(".java")) { + String fileName = sourceCodeFile.getName(); + fileName = fileName.substring(8).replace(".java", ".txt"); + File positionFile = new File(inputPath + "UnFV_positions/" + violationType + "/" + fileName); // position file + MessageFile msgFile = new MessageFile(null, sourceCodeFile, null); + msgFile.setPositionFile(positionFile); + msgFiles.add(msgFile); + } + } + + return msgFiles; + } +} diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser3/ParseFixPatternActor.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser3/ParseFixPatternActor.java new file mode 100644 index 0000000..1788e9c --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser3/ParseFixPatternActor.java @@ -0,0 +1,80 @@ +package edu.lu.uni.serval.MultipleThreadsParser3; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import akka.actor.ActorRef; +import akka.actor.Props; +import akka.actor.UntypedActor; +import akka.japi.Creator; +import akka.routing.RoundRobinPool; +import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; +import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage; + +public class ParseFixPatternActor extends UntypedActor { + + private static Logger logger = LoggerFactory.getLogger(ParseFixPatternActor.class); + + private ActorRef mineRouter; + private final int numberOfWorkers; + private int counter = 0; + + public ParseFixPatternActor(int numberOfWorkers, String sourceCodeFilesPath, String vType) { + mineRouter = this.getContext().actorOf(new RoundRobinPool(numberOfWorkers) + .props(ParseFixPatternWorker.props(sourceCodeFilesPath)), "mine-pattern-router-" + vType); + this.numberOfWorkers = numberOfWorkers; + } + + public static Props props(final int numberOfWorkers, final String sourceCodeFilesPath, final String vType) { + + return Props.create(new Creator() { + + private static final long serialVersionUID = 9207427376110704705L; + + @Override + public ParseFixPatternActor create() throws Exception { + return new ParseFixPatternActor(numberOfWorkers, sourceCodeFilesPath, vType); + } + + }); + } + + @SuppressWarnings("deprecation") + @Override + public void onReceive(Object message) throws Exception { + if (message instanceof WorkMessage) { + List files = ((WorkMessage) message).getMsgFiles(); + int size = files.size(); + int average = (int) Math.round((double) size / numberOfWorkers); + + for (int i = 0; i < numberOfWorkers; i ++) { + int fromIndex = i * average; + int toIndex = (i + 1) * average; + if (i == numberOfWorkers - 1) { + toIndex = size; + } + + List filesOfWorkers = new ArrayList<>(); + filesOfWorkers.addAll(files.subList(fromIndex, toIndex)); + final WorkMessage workMsg = new WorkMessage(i + 1, filesOfWorkers); + mineRouter.tell(workMsg, getSelf()); + logger.info("Assign a task to worker #" + (i + 1) + "..."); + } + } else if ("STOP".equals(message.toString())) { + counter ++; + logger.info(counter + " workers finished their work..."); + if (counter >= numberOfWorkers) { + logger.info("All workers finished their work..."); + this.getContext().stop(mineRouter); + this.getContext().stop(getSelf()); + this.getContext().system().shutdown(); + } + } else { + unhandled(message); + } + } + +} diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser3/ParseFixPatternWorker.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser3/ParseFixPatternWorker.java new file mode 100644 index 0000000..94f9bdf --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser3/ParseFixPatternWorker.java @@ -0,0 +1,96 @@ +package edu.lu.uni.serval.MultipleThreadsParser3; + +import java.io.File; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import akka.actor.Props; +import akka.actor.UntypedActor; +import akka.japi.Creator; +import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; +import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage; +import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.violation.code.parser.ViolationCodeParser; + +public class ParseFixPatternWorker extends UntypedActor { + private static Logger log = LoggerFactory.getLogger(ParseFixPatternActor.class); + + private String sourceCodeFilesPath; + + public ParseFixPatternWorker(String sourceCodeFilesPath) { + this.sourceCodeFilesPath = sourceCodeFilesPath; + } + + public static Props props(final String sourceCodeFilesPath) { + return Props.create(new Creator() { + + private static final long serialVersionUID = -7615153844097275009L; + + @Override + public ParseFixPatternWorker create() throws Exception { + return new ParseFixPatternWorker(sourceCodeFilesPath); + } + + }); + } + + @Override + public void onReceive(Object message) throws Exception { + if (message instanceof WorkMessage) { + WorkMessage msg = (WorkMessage) message; + List files = msg.getMsgFiles(); + StringBuilder sourceCode = new StringBuilder(); + StringBuilder sizes = new StringBuilder(); + StringBuilder tokens = new StringBuilder(); + + int id = msg.getId(); + int counter = 0; + + for (MessageFile msgFile : files) { + File prevFile = msgFile.getPrevFile(); + File positionFile = msgFile.getPositionFile(); + if (prevFile.getName().toLowerCase().contains("test")) { + continue; + } + ViolationCodeParser parser = new ViolationCodeParser(); + parser.parse(prevFile, positionFile); + + String sourceCodeStr = parser.sourceCode; + if ("".equals(sourceCodeStr)) { + } else { + sourceCode.append(sourceCodeStr); + sizes.append(parser.sizes); + tokens.append(parser.tokens); + + counter ++; + if (counter % 100 == 0) { + FileHelper.outputToFile(sourceCodeFilesPath + "SourCode/sourcecode_" + id + ".list", sourceCode, true); + FileHelper.outputToFile(sourceCodeFilesPath + "Sizes/sourcecode_" + id + ".list", sizes, true); + FileHelper.outputToFile(sourceCodeFilesPath + "Tokens/sourcecode_" + id + ".list", tokens, true); + sourceCode.setLength(0); + sizes.setLength(0); + tokens.setLength(0); + } + } + } + + if (sizes.length() > 0) { + FileHelper.outputToFile(sourceCodeFilesPath + "SourCode/sourcecode_" + id + ".list", sourceCode, true); + FileHelper.outputToFile(sourceCodeFilesPath + "Sizes/sourcecode_" + id + ".list", sizes, true); + FileHelper.outputToFile(sourceCodeFilesPath + "Tokens/sourcecode_" + id + ".list", tokens, true); + sourceCode.setLength(0); + sizes.setLength(0); + tokens.setLength(0); + } + + log.info("Worker #" + id +"Finish of parsing " + counter + " files..."); + log.info("Worker #" + id + " finished the work..."); + this.getSender().tell("STOP", getSelf()); + } else { + unhandled(message); + } + } + +} diff --git a/src/main/java/edu/lu/uni/serval/config/Configuration.java b/src/main/java/edu/lu/uni/serval/config/Configuration.java index afa2ce3..90eb62a 100644 --- a/src/main/java/edu/lu/uni/serval/config/Configuration.java +++ b/src/main/java/edu/lu/uni/serval/config/Configuration.java @@ -2,7 +2,7 @@ package edu.lu.uni.serval.config; public class Configuration { - public static final long SECONDS_TO_WAIT = 30L; + public static final long SECONDS_TO_WAIT = 300000L; public static final String ROOT_PATH = "../FPM_Violations/"; // The root path of all output data. diff --git a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java b/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java index cd057fe..c3a1bac 100644 --- a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java +++ b/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java @@ -186,7 +186,17 @@ public class HierarchicalRegrouper { private Action findParentAction(Action action, List actions) { ITree parent = action.getNode().getParent(); - if (parent.getType() == 55) return null; + if (parent.getType() == 55) { + int type = action.getNode().getType(); + // Modifier, NormalAnnotation, MarkerAnnotation, SingleMemberAnnotation + if (type != 83 && type != 77 && type != 78 && type != 79 + && type != 5 && type != 39 && type != 43 && type != 74 && type != 75 + && type != 76 && type != 84 && type != 87 && type != 88 && type != 42) { + // ArrayType, PrimitiveType, SimpleType, ParameterizedType, + // QualifiedType, WildcardType, UnionType, IntersectionType, NameQualifiedType, SimpleName + return null; + } + } if (action instanceof Addition) { parent = ((Addition) action).getParent(); // parent in the fixed source code tree diff --git a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java b/src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java index f3accce..c884ea3 100644 --- a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java +++ b/src/main/java/edu/lu/uni/serval/gumtree/regroup/HunkActionFilter.java @@ -193,7 +193,8 @@ public class HunkActionFilter { private int getEndPosition(List children) { int endPosition = 0; - for (ITree child : children) { + for (int i = 0, size = children.size(); i < size; i ++) { + ITree child = children.get(i); int type = child.getType(); if (type == 6 || type == 10 || type == 12 || type == 17 || type == 18 || type == 19 || type == 21 || type == 8// Block, EmptyStatement || type == 24 || type == 25 || type == 30 || type == 41 || type == 46 || type == 49 || type == 50 @@ -202,7 +203,12 @@ public class HunkActionFilter { // ExpressionStatement, ForStatement, IfStatement, LabeledStatement, ReturnStatement, SuperConstructorInvocation // SwitchCase, SwitchStatement, SynchronizedStatement, ThrowStatement, TryStatement // TypeDeclarationStatement, VariableDeclarationStatement, WhileStatement, EnhancedForStatement - endPosition = child.getPos() - 1; + if ( i > 0) { + child = children.get(i - 1); + endPosition = child.getPos() + child.getLength(); + } else { + endPosition = child.getPos() - 1; + } break; } } @@ -372,6 +378,12 @@ public class HunkActionFilter { CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); if (prevUnit == null || revUnit == null) { + for (Violation violation : violations) { + this.unfixedViolations += "#NullMatchedGumTreeResult:" + revFile.getName() + ":" +violation.getStartLineNum() + ":" + + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; + System.err.println("#NullMatchedGumTreeResult:" + revFile.getName() + ":" +violation.getStartLineNum() + ":" + + violation.getEndLineNum() + ":" + violation.getViolationType()); + } return selectedViolations; } @@ -383,7 +395,7 @@ public class HunkActionFilter { // String type = getType(violation); // continue; } else if (bugHunkStartLine == -1) {// - specialVioaltionTypes(violation, actionSets); + specialVioaltionTypes(violation, actionSets, prevUnit, revUnit); // continue; } else { int bugHunkEndLine = violation.getBugEndLineNum(); @@ -431,17 +443,24 @@ public class HunkActionFilter { // violation.getActionSets().addAll(matchedActionSets); selectedViolations.add(violation); } else { + this.unfixedViolations += "#NullMatchedGumTreeResult:" + revFile.getName() + ":" +violation.getStartLineNum() + ":" + + violation.getEndLineNum() + ":" + violation.getViolationType() + "\n"; System.err.println("#NullMatchedGumTreeResult:" + revFile.getName() + ":" +violation.getStartLineNum() + ":" + violation.getEndLineNum() + ":" + violation.getViolationType()); } } return selectedViolations; } + public String unfixedViolations = ""; - private void specialVioaltionTypes(Violation violation, List actionSets) { + private void specialVioaltionTypes(Violation violation, List actionSets, CompilationUnit prevUnit, CompilationUnit revUnit) { String type = violation.getViolationType(); if ("SE_NO_SUITABLE_CONSTRUCTOR".equals(type)) {// 158, 47 for (HierarchicalActionSet actionSet : actionSets) { + int actionBugStartLine = actionSet.getBugStartLineNum(); + if (actionBugStartLine == 0) { + actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); + } if (actionSet.getActionString().startsWith("UPD TypeDeclaration@@")) { violation.getActionSets().add(actionSet); break; @@ -450,6 +469,10 @@ public class HunkActionFilter { } else if ("CN_IDIOM".equals(type)) { // 202 23 //add clone method. or update clone method for (HierarchicalActionSet actionSet : actionSets) { + int actionBugStartLine = actionSet.getBugStartLineNum(); + if (actionBugStartLine == 0) { + actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); + } if (actionSet.getActionString().startsWith("INS MethodDeclaration@@clone")) { // || actionSet.getActionString().startsWith("UPD MethodDeclaration@@clone")) { violation.getActionSets().add(actionSet); @@ -459,6 +482,10 @@ public class HunkActionFilter { } else if ("SE_NO_SERIALVERSIONID".equals(type)) {// 12 1960 // change superclass or interface, add field or remove @SuppressWarnings("serial"), some are inner class for (HierarchicalActionSet actionSet : actionSets) { + int actionBugStartLine = actionSet.getBugStartLineNum(); + if (actionBugStartLine == 0) { + actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); + } if (actionSet.getActionString().startsWith("UPD TypeDeclaration@")) { violation.getActionSets().add(actionSet); } @@ -470,6 +497,10 @@ public class HunkActionFilter { } else if ("SE_NO_SUITABLE_CONSTRUCTOR_FOR_EXTERNALIZATION".equals(type)) { // 175 34 // constructor for (HierarchicalActionSet actionSet : actionSets) { + int actionBugStartLine = actionSet.getBugStartLineNum(); + if (actionBugStartLine == 0) { + actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); + } if (actionSet.getActionString().startsWith("UPD MethodDeclaration@@")) { violation.getActionSets().add(actionSet); break; @@ -478,6 +509,10 @@ public class HunkActionFilter { } else if ("SE_COMPARATOR_SHOULD_BE_SERIALIZABLE".equals(type)) { // 89 148 //class, and add a field se...? for (HierarchicalActionSet actionSet : actionSets) { + int actionBugStartLine = actionSet.getBugStartLineNum(); + if (actionBugStartLine == 0) { + actionBugStartLine = setLineNumbers(actionSet, prevUnit, revUnit); + } if (actionSet.getActionString().startsWith("UPD TypeDeclaration@")) { violation.getActionSets().add(actionSet); } @@ -568,108 +603,6 @@ public class HunkActionFilter { return null; } - private boolean isRanged(HierarchicalActionSet actionSet, Violation violation) { - int actionStartLine = actionSet.getFixStartLineNum(); - int actionEndLine = actionSet.getFixEndLineNum(); - int violationStartLine = violation.getStartLineNum(); - int violationEndLine = violation.getEndLineNum(); - List hunks = violation.getHunks(); - - for (DiffEntryHunk hunk : hunks) { - int bugStartLine = hunk.getBugLineStartNum(); - int bugEndLine = bugStartLine + hunk.getBugRange(); - int fixStartLine = hunk.getFixLineStartNum(); - int fixEndLine = fixStartLine + hunk.getFixRange(); -// if (fixStartLine > actionEndLine || bugStartLine > violationEndLine) break; - if (fixEndLine < actionStartLine || bugEndLine < violationStartLine) continue; - - String hunkContent = hunk.getHunk(); - BufferedReader reader = null; - int counterOfContext = 0; - int counterOfDeletedLines = 0; - int counterOfAddedLines = 0; - int bugStarts = 0; - int bugEnds = 0; - int fixStarts = 0; - int fixEnds = 0; - int contextStarts = 0; - try { - reader = new BufferedReader(new StringReader(hunkContent)); - String line = null; - while ((line = reader.readLine()) != null) { - if (line.startsWith("-")) { - counterOfDeletedLines ++; - if (bugStarts == 0) { - bugStarts = bugStartLine + counterOfContext + counterOfDeletedLines - 1; - } - contextStarts = 0; - - if (fixStarts != 0) { - fixEnds = fixStartLine + counterOfContext + counterOfAddedLines - 1; - if (fixStarts > actionEndLine) break; - if (fixEnds < actionStartLine) { - fixStarts = 0; - continue; - } - return true; - } - } - else if (line.startsWith("+")) { - counterOfAddedLines ++; - if (bugStarts == 0) { - bugStarts = contextStarts; - } - bugEnds = bugStartLine + counterOfContext + counterOfDeletedLines - 1; - if (violationEndLine < bugStarts) break; - if (violationStartLine > bugEnds) { - bugStarts = 0; - } - if (bugStarts != 0 && fixStarts == 0) { - fixStarts = fixStartLine + counterOfContext + counterOfAddedLines - 1; - } - } - else { - counterOfContext ++; - bugStarts = 0; - if (contextStarts == 0) { - contextStarts = bugStartLine + counterOfContext + counterOfDeletedLines - 1; - } - - if (fixStarts != 0) { - fixEnds = fixStartLine + counterOfContext + counterOfAddedLines - 1; - if (fixStarts > actionEndLine) break; - if (fixEnds < actionStartLine) { - fixStarts = 0; - continue; - } - - return true; - } - } - } - - if (fixStarts != 0) { - fixEnds = fixStartLine + counterOfContext + counterOfAddedLines - 1; - if (fixStarts > actionEndLine) break; - if (fixEnds < actionStartLine) { - fixStarts = 0; - continue; - } - return true; - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - reader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - return false; - } - /** * Filter out the modify actions, which are not in the DiffEntry hunks, without considering DiffEntry hunks. * @@ -821,9 +754,9 @@ public class HunkActionFilter { // Modifier, NormalAnnotation, MarkerAnnotation, SingleMemberAnnotation if (type != 83 && type != 77 && type != 78 && type != 79 && type != 5 && type != 39 && type != 43 && type != 74 && type != 75 - && type != 76 && type != 84 && type != 87 && type != 88) { + && type != 76 && type != 84 && type != 87 && type != 88 && type != 42) { // ArrayType, PrimitiveType, SimpleType, ParameterizedType, - // QualifiedType, WildcardType, UnionType, IntersectionType, NameQualifiedType + // QualifiedType, WildcardType, UnionType, IntersectionType, NameQualifiedType, SimpleName if (i > 0) { child = children.get(i - 1); return child.getPos() + child.getLength() + 1; diff --git a/src/main/java/edu/lu/uni/serval/gumtree/regroup/SimplifyTree.java b/src/main/java/edu/lu/uni/serval/gumtree/regroup/SimplifyTree.java index 2d43aa2..ee9950d 100644 --- a/src/main/java/edu/lu/uni/serval/gumtree/regroup/SimplifyTree.java +++ b/src/main/java/edu/lu/uni/serval/gumtree/regroup/SimplifyTree.java @@ -178,12 +178,17 @@ public class SimplifyTree { if (astNode.endsWith("Type")) { simpleTree.setLabel(canonicalizeTypeStr(label).replaceAll(" ", "")); } else { - if ((astNode.equals("SimpleName") || astNode.equals("MethodInvocation")) && label.startsWith("MethodName:")) { - simpleTree.setNodeType("MethodName"); - label = label.substring(11); - int argusIndex = label.indexOf(":["); - if (argusIndex > 0) { - label = label.substring(0, argusIndex); + if ((astNode.equals("SimpleName") || astNode.equals("MethodInvocation"))) { + if (label.startsWith("MethodName:")) { + simpleTree.setNodeType("MethodName"); + label = label.substring(11); + int argusIndex = label.indexOf(":["); + if (argusIndex > 0) { + label = label.substring(0, argusIndex); + } + } else if (label.startsWith("ClassName:")) { + simpleTree.setNodeType("ClassName"); + label = label.substring(10); } simpleTree.setLabel(label); } else { @@ -206,6 +211,10 @@ public class SimplifyTree { label = label.substring(0, argusIndex); } simpleTree.setLabel(label); + } else if (label.startsWith("ClassName:")) { + simpleTree.setNodeType("ClassName"); + label = label.substring(10); + simpleTree.setLabel(label); } else if (label.startsWith("Name:")) { label = label.substring(5); char firstChar = label.charAt(0); @@ -256,12 +265,17 @@ public class SimplifyTree { if (astNode.endsWith("Type")) { simpleTree.setLabel(canonicalizeTypeStr(label).replaceAll(" ", "")); } else { - if ((astNode.equals("SimpleName") || astNode.equals("MethodInvocation")) && label.startsWith("MethodName:")) { - simpleTree.setNodeType("MethodName"); - label = label.substring(11); - int argusIndex = label.indexOf(":["); - if (argusIndex > 0) { - label = label.substring(0, argusIndex); + if ((astNode.equals("SimpleName") || astNode.equals("MethodInvocation"))) { + if (label.startsWith("MethodName:")) { + simpleTree.setNodeType("MethodName"); + label = label.substring(11); + int argusIndex = label.indexOf(":["); + if (argusIndex > 0) { + label = label.substring(0, argusIndex); + } + } else if (label.startsWith("ClassName:")) { + simpleTree.setNodeType("ClassName"); + label = label.substring(10); } simpleTree.setLabel(label); } else { @@ -285,6 +299,10 @@ public class SimplifyTree { label = label.substring(0, argusIndex); } simpleTree.setLabel(label); + } else if (label.startsWith("ClassName:")) { + simpleTree.setNodeType("ClassName"); + label = label.substring(10); + simpleTree.setLabel(label); } else if (label.startsWith("Name:")) { label = label.substring(5); char firstChar = label.charAt(0); diff --git a/src/main/java/edu/lu/uni/serval/statistics/Statistic.java b/src/main/java/edu/lu/uni/serval/statistics/Statistic.java index 3bfb754..ea6527f 100644 --- a/src/main/java/edu/lu/uni/serval/statistics/Statistic.java +++ b/src/main/java/edu/lu/uni/serval/statistics/Statistic.java @@ -55,6 +55,8 @@ public class Statistic { String statistic = "../FPM_Violations/OUTPUT"; List files = FileHelper.getAllFiles(statistic, ".list"); + int positions = 0; + int numV = 0; int testAlarms = 0; int nullGumTreeResults = 0; int nullMappingGumTreeResults = 0; @@ -67,112 +69,167 @@ public class Statistic { int nullDiffentry = 0; int TestingInfo = 0; int i = 0; + Map types1 = new HashMap<>(); for (File file : files) { - if (file.getName().startsWith("test")) continue; - String content = FileHelper.readFile(file); - BufferedReader reader = new BufferedReader(new StringReader(content)); - String line = null; - i ++; - try { - while ((line = reader.readLine()) != null) { - if (line.startsWith("TestViolations")) { - testAlarms += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); - } else if (line.startsWith("NullGumTreeResults")) { - nullGumTreeResults += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); - } else if (line.startsWith("NoSourceCodeChanges")) { - noSourceCodeChagnes += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); - } else if (line.startsWith("NoStatementChanges")) { - noStatementChanges += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); - } else if (line.startsWith("NullDiffEntry")) { - nullDiffentry += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); - } else if (line.startsWith("NullMatchedGumTreeResults")) { - nullMappingGumTreeResults += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); - } else if (line.startsWith("PureDeletion")) { - pureDeletion += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); - } else if (line.startsWith("LargeHunk")) { - largeHunk += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); - } else if (line.startsWith("NullSourceCode")) { - nullSourceCode += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); - } else if (line.startsWith("Timeout")) { - timeout += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); - } else if (line.startsWith("TestingInfo")) { - TestingInfo +=Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); - } - } - } catch (IOException e) { - e.printStackTrace(); - } finally { + if (file.getName().startsWith("statistic")) { + String content = FileHelper.readFile(file); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + i ++; try { - reader.close(); + while ((line = reader.readLine()) != null) { + if (line.startsWith("Positions")) { + positions += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("NumViolations")) { + numV += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else + if (line.startsWith("TestViolations")) { + testAlarms += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("NullGumTreeResults")) { + nullGumTreeResults += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("NoSourceCodeChanges")) { + noSourceCodeChagnes += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("NoStatementChanges")) { + noStatementChanges += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("NullDiffEntry")) { + nullDiffentry += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("NullMatchedGumTreeResults")) { + nullMappingGumTreeResults += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("PureDeletion")) { + pureDeletion += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("LargeHunk")) { + largeHunk += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("NullSourceCode")) { + nullSourceCode += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("Timeout")) { + timeout += Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } else if (line.startsWith("TestingInfo")) { + TestingInfo +=Integer.parseInt(line.substring(line.lastIndexOf(":") + 1).trim()); + } + } } catch (IOException e) { e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + + if (file.getName().startsWith("UnfixedV")) { + String content = FileHelper.readFile(file); + BufferedReader reader = new BufferedReader(new StringReader(content)); + String line = null; + i ++; + try { + while ((line = reader.readLine()) != null) { +// types1 + if (line.startsWith("## OAR [")) break; + String type = line.substring(0, line.indexOf(":")); + if (types1.containsKey(type)) { + types1.put(type, types1.get(type) + 1); + } else { + types1.put(type, 1); + } + } + } catch(IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } } } } - // 22280, 23275 -// Statistics: -// TestViolation: 4682 -// NullGumTreeResults: 0 -// NoSourceCodeChange: 5995 -// NoStatementChange: 0 -// NullDiffEntry: 6450 -// NullMatchedGumTreeResult: 15675 -// PureDeletion: 15740 -// LargeHunk: 13521 -// NullSourceCode: 0 -// Timeout: 7205 -// TestingInfo: 8703 -// 62231 22634 -// #PureDeletion1: 16864 -// #NullMatchedGumTreeResult: 17511 -// #NullDiffEntry: 6986 -// #LargeHunk: 14625 -// #NoSourceCodeChange: 7010 -// #TestingInfo: 9071 -// #TestViolation: 4682 -// #Timeout: 7205 -// 67090 22634 - - System.out.println(i); - System.out.println("\n\nStatistics:\nTestViolation: " + testAlarms); - System.out.println("NullGumTreeResults: " + nullGumTreeResults); - System.out.println("NoSourceCodeChange: " + noSourceCodeChagnes); - System.out.println("NoStatementChange: " + noStatementChanges); - System.out.println("NullDiffEntry: " + nullDiffentry); - System.out.println("NullMatchedGumTreeResult: " + nullMappingGumTreeResults); - System.out.println("PureDeletion: " + pureDeletion); - System.out.println("LargeHunk: " + largeHunk); - System.out.println("NullSourceCode: " + nullSourceCode); - System.out.println("Timeout: " + timeout); - System.out.println("TestingInfo: " + TestingInfo); - System.out.println(testAlarms + nullGumTreeResults + noSourceCodeChagnes + noStatementChanges + - nullDiffentry + nullMappingGumTreeResults + largeHunk + nullSourceCode + timeout + TestingInfo); - Map types = new HashMap<>(); - FileInputStream fis = new FileInputStream("../FPM_Violations/OAR.FPM.4222170.stderr"); - Scanner scanner = new Scanner(fis); - while (scanner.hasNextLine()) { - String line = scanner.nextLine(); - if (line.startsWith("## OAR [")) break; - String type = line.substring(0, line.indexOf(":")); - if (types.containsKey(type)) { - types.put(type, types.get(type) + 1); - } else { - types.put(type, 1); - } - } - scanner.close(); - fis.close(); - - int sum = 0; - for (Map.Entry entry : types.entrySet()) { +// FileInputStream fis = new FileInputStream("../FPM_Violations/OAR.FPM.4222208.stderr"); +// Scanner scanner = new Scanner(fis); +// while (scanner.hasNextLine()) { +// String line = scanner.nextLine(); +// if (line.startsWith("## OAR [")) break; +// String type = line.substring(0, line.indexOf(":")); +// if (types.containsKey(type)) { +// types.put(type, types.get(type) + 1); +// } else { +// types.put(type, 1); +// } +// } +// scanner.close(); +// fis.close(); +// int sum = 0; +// int sum2 = 0; +// for (Map.Entry entry : types.entrySet()) { +// System.out.println(entry.getKey() + ": " + entry.getValue()); +// if (!entry.getKey().startsWith("#PureDeletion")) { +// if (!entry.getKey().startsWith("#Timeout") && !entry.getKey().startsWith("#TestViolation")) +// sum += entry.getValue(); +// else sum2 += entry.getValue(); +// } +// } + int sum3 = 0; + int sum4 = 0; + for (Map.Entry entry : types1.entrySet()) { System.out.println(entry.getKey() + ": " + entry.getValue()); if (!entry.getKey().startsWith("#PureDeletion")) { - sum += entry.getValue(); + if (!entry.getKey().startsWith("#Timeout") && !entry.getKey().startsWith("#TestViolation")) + sum3 += entry.getValue(); + else sum4 += entry.getValue(); } } - System.out.println(sum); + + System.out.println(i); + System.out.println("\n\nStatistics:\nPositions: " + positions); + System.out.println("NumViolations: " + numV); + System.out.println("\nTestViolation: " + testAlarms + " :: " + types.get("#TestViolation") + " :: " + types1.get("#TestViolation")); + System.out.println("NullGumTreeResults: " + nullGumTreeResults + " :: " + types.get("#NullGumTreeResults") + " :: " + types1.get("#NullGumTreeResults")); + System.out.println("NoSourceCodeChange: " + noSourceCodeChagnes + " :: " + types.get("#NoSourceCodeChange") + " :: " + types1.get("#NoSourceCodeChange")); + System.out.println("NoStatementChange: " + noStatementChanges + " :: " + types.get("#NoStatementChange") + " :: " + types1.get("#NoStatementChange")); + System.out.println("NullDiffEntry: " + nullDiffentry + " :: " + types.get("#NullDiffEntry") + " :: " + types1.get("#NullDiffEntry")); + System.out.println("NullMatchedGumTreeResult: " + nullMappingGumTreeResults + " :: " + types.get("#NullMatchedGumTreeResult") + " :: " + types1.get("#NullMatchedGumTreeResult")); + System.out.println("PureDeletion: " + pureDeletion + " :: " + types.get("#PureDeletion") + " :: " + types1.get("#PureDeletion")); + System.out.println("LargeHunk: " + largeHunk + " :: " + types.get("#LargeHunk") + " :: " + types1.get("#LargeHunk")); + System.out.println("NullSourceCode: " + nullSourceCode + " :: " + types.get("#NullSourceCode") + " :: " + types1.get("#NullSourceCode")); + System.out.println("Timeout: " + timeout + " :: " + types.get("#Timeout") + " :: " + types1.get("#Timeout")); + System.out.println("TestingInfo: " + TestingInfo + " :: " + types.get("#TestingInfo") + " :: " + types1.get("#TestingInfo")); + System.out.println("A: " + (positions + testAlarms + timeout)); + System.out.println("B: " + (numV + testAlarms + timeout)); + System.out.println(testAlarms + nullGumTreeResults + noSourceCodeChagnes + noStatementChanges + + nullDiffentry + nullMappingGumTreeResults + largeHunk + nullSourceCode + timeout + TestingInfo); + System.out.println(nullGumTreeResults + noSourceCodeChagnes + noStatementChanges + + nullDiffentry + nullMappingGumTreeResults + largeHunk + nullSourceCode + TestingInfo); + System.out.println(testAlarms + timeout); +// System.out.println(sum); +// System.out.println(sum2); + System.out.println(sum3); + System.out.println(sum4); + /*30717 56630 1580, 88927 + * Statistics: + +TestViolation: 4682 :: null :: null +NullGumTreeResults: 0 :: null :: null +NoSourceCodeChange: 7010 :: null :: 7010 +NoStatementChange: 0 :: null :: null +NullDiffEntry: 7100 :: null :: 7100 +NullMatchedGumTreeResult: 25336 :: null :: 25336 +PureDeletion: 18238 :: null :: 18238 +LargeHunk: 12502 :: null :: 12502 +NullSourceCode: 0 :: null :: null +Timeout: 0 :: null :: null +TestingInfo: 9071 :: null :: 9071 +A: 88927 +B: 88927 +63759 +59077 +4682 +59077 +0 + */ } public static void statistics(String fileName, String type) throws IOException { diff --git a/src/main/java/edu/lu/uni/serval/violation/code/parser/Test.java b/src/main/java/edu/lu/uni/serval/violation/code/parser/Test.java index 990c973..426d538 100644 --- a/src/main/java/edu/lu/uni/serval/violation/code/parser/Test.java +++ b/src/main/java/edu/lu/uni/serval/violation/code/parser/Test.java @@ -21,8 +21,8 @@ public class Test { public static void main(String[] args) throws IOException { // testV1(); // testV2(); -// testV3("../FPM_Violations/OAR.FPM.4222152.stderr", "OUTPUT/unparsedviolations.txt"); - testV4("../FPM_Violations/OAR.FPM.4222152.stderr"); + testV3("../FPM_Violations/OAR.FPM.4222306.stderr", "OUTPUT/unparsedviolations.txt"); +// testV4("../FPM_Violations/OAR.FPM.4222208.stderr"); } private static void testV4(String inputFile) throws IOException { @@ -91,7 +91,10 @@ public class Test { String type = elements[0]; if (types.containsKey(type)) { - types.get(type).add(elements[1] + " : " + elements[2] + " : " + elements[3] + " : " + elements[4]); + String info = elements[1] + " : " + elements[2] + " : " + elements[3] + " : " + elements[4]; + if (!types.get(type).contains(info)) { + types.get(type).add(info); + } } else { List files = new ArrayList<>(); files.add(elements[1] + " : " + elements[2] + " : " + elements[3] + " : " + elements[4]); @@ -104,10 +107,12 @@ public class Test { StringBuilder builder = new StringBuilder(); for (Map.Entry> entry : types.entrySet()) { System.out.println(entry.getKey()); - builder.append(entry.getKey() + "\n"); - List files = entry.getValue(); - for (String file : files) { - builder.append(" " + file + "\n"); + if (entry.getKey().equals("#LargeHunk") || entry.getKey().equals("#TestingInfo")) { + builder.append(entry.getKey() + "\n"); + List files = entry.getValue(); + for (String file : files) { + builder.append(" " + file + "\n"); + } } } FileHelper.outputToFile(outputFile, builder, false); diff --git a/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolatePositionReader.java b/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolatePositionReader.java new file mode 100644 index 0000000..21adc92 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolatePositionReader.java @@ -0,0 +1,115 @@ +package edu.lu.uni.serval.violation.code.parser; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.utils.FileHelper; + +/** + * Read violations' positions. + * + * Class name part, + * Field declaration, + * Initializer, + * Method name, + * Method body, + * Inner class, + * Anonymous class. + * + * @author kui.liu + * + */ +public class ViolatePositionReader { + /* +-1 : 3788 +0 : 1921 +1 : 3106 +71 : 302 +23 : 4929 +72 : 87 +28 : 404 +31 : 74390 + */ + public static void main(String[] args) throws NumberFormatException, IOException { + List msgFiles = getMessageFiles(Configuration.GUM_TREE_INPUT); + + Map map1 = new HashMap<>(); + Map> map2 = new HashMap<>(); + for (MessageFile file : msgFiles) { + String fileContent = FileHelper.readFile(file.getPositionFile()); + BufferedReader reader = null; + reader = new BufferedReader(new StringReader(fileContent)); + String line = null; + + while ((line = reader.readLine()) != null) { + String[] positionStr = line.split(":"); + int startLine = Integer.parseInt(positionStr[1]); + int endLine = Integer.parseInt(positionStr[2]); + String violationType = positionStr[0]; + + ViolationSourceCodeTree alarmTree = new ViolationSourceCodeTree(file.getPrevFile(), startLine, endLine); + alarmTree.locateParentNode(); + + int range = alarmTree.getViolationFinalStartLine(); + if (map1.containsKey(range)) { + map1.put(range, map1.get(range) + 1); + } else { + map1.put(range, 1); + } + + if (map2.containsKey(violationType)) { + Map map = map2.get(violationType); + if (map.containsKey(range)) { + map.put(range, map.get(range) + 1); + } else { + map.put(range, 1); + } + } else { + Map map = new HashMap<>(); + map.put(range, 1); + map2.put(violationType, map); + } + } + + reader.close(); + } + for (Map.Entry entry : map1.entrySet()) { + System.out.println(entry.getKey() + " : " + entry.getValue()); + } + } + + /** + * Get violation-related files. + * + * @param gumTreeInput + * @return + */ + public static List getMessageFiles(String gumTreeInput) { + String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile + File revFilesPath = new File(inputPath + "revFiles/"); + File[] revFiles = revFilesPath.listFiles(); // project folders + List msgFiles = new ArrayList<>(); + + for (File revFile : revFiles) { + if (revFile.getName().endsWith(".java")) { + String fileName = revFile.getName(); + File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file + fileName = fileName.replace(".java", ".txt"); + File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file + MessageFile msgFile = new MessageFile(null, prevFile, null); + msgFile.setPositionFile(positionFile); + msgFiles.add(msgFile); + } + } + + return msgFiles; + } +} diff --git a/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolationCodeParser.java b/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolationCodeParser.java index e42f5fc..b24276f 100644 --- a/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolationCodeParser.java +++ b/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolationCodeParser.java @@ -2,10 +2,13 @@ package edu.lu.uni.serval.violation.code.parser; import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.List; +import java.util.Scanner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,6 +113,84 @@ public class ViolationCodeParser { FileHelper.outputToFile(outputPath, tokensBuilder, true); tokensBuilder.setLength(0); } + + public String sourceCode = ""; + public String tokens = ""; + public String sizes = ""; + + public void parse(File javaFile, File positionFile) { + + List violations = readViolationInfo(positionFile); + for (Violation violation : violations) { + int startLine = violation.getStartLineNum(); + int endLine = violation.getEndLineNum(); + + if (endLine > startLine + 5) { +// log.warn("#Large_Violation_Hunk: " + javaFile.getName() + ":" + startLine + ":" + endLine + ":" + alarmType); + continue; + } + + ViolationSourceCodeTree alarmTree = new ViolationSourceCodeTree(javaFile, startLine, endLine); + alarmTree.extract(); + List matchedTrees = alarmTree.getViolationSourceCodeTrees(); + if (matchedTrees.size() == 0) { + System.err.println("#Null_Violation_Hunk: " + javaFile.getName() + ":" + startLine + ":" + endLine); + continue; + } + SimpleTree simpleTree = new SimpleTree(); + simpleTree.setLabel("Block"); + simpleTree.setNodeType("Block"); + List children = new ArrayList<>(); + + for (ITree matchedTree : matchedTrees) { + SimpleTree simpleT = new SimplifyTree().canonicalizeSourceCodeTree(matchedTree, null); + children.add(simpleT); + } + simpleTree.setChildren(children); + + String tokens = Tokenizer.getTokensDeepFirst(simpleTree); + String[] tokensArray = tokens.split(" "); + int length = tokensArray.length; + sizes += length + "\n"; + this.tokens += tokens + "\n"; + + startLine = alarmTree.getViolationFinalStartLine(); + endLine = alarmTree.getViolationFinalEndLine(); + String sourceCode = readSourceCode(javaFile, startLine, endLine); + this.sourceCode += sourceCode + "\n"; + } + } + + private String readSourceCode(File javaFile, int startLine, int endLine) { + StringBuilder sourceCode = new StringBuilder("##Source_Code:\n"); + FileInputStream fis = null; + Scanner scanner = null; + + try { + fis = new FileInputStream(javaFile); + scanner = new Scanner(fis); + int counter = 0; + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + counter ++; + if (startLine <= counter && counter <= endLine) { + sourceCode.append(line + "\n"); + } + if (counter == endLine) break; + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + try { + scanner.close(); + fis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + return sourceCode.toString(); + } private List readViolationInfo(String file) { List violations = new ArrayList<>(); @@ -144,4 +225,37 @@ public class ViolationCodeParser { } return violations; } + + private List readViolationInfo(File file) { + List violations = new ArrayList<>(); + + String fileContent = FileHelper.readFile(file); + BufferedReader reader = null; + reader = new BufferedReader(new StringReader(fileContent)); + String line = null; + try { + while ((line = reader.readLine()) != null) { + String[] positionStr = line.split(":"); + int startLine = Integer.parseInt(positionStr[1]); + int endLine = Integer.parseInt(positionStr[2]); + String alarmType = positionStr[0]; + + if (startLine == -1 || endLine == -1) { + continue; + } + + Violation violation = new Violation(startLine, endLine, alarmType); + violations.add(violation); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return violations; + } } diff --git a/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolationSourceCodeTree.java b/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolationSourceCodeTree.java index 32b83ae..068ce50 100644 --- a/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolationSourceCodeTree.java +++ b/src/main/java/edu/lu/uni/serval/violation/code/parser/ViolationSourceCodeTree.java @@ -50,6 +50,9 @@ public class ViolationSourceCodeTree { return violationFinalEndLine; } + /** + * extract source code of violations in method body or field body. + */ public void extract() { ITree rootTree = new GumTreeGenerator().generateITreeForJavaFile(file, GumTreeType.EXP_JDT); @@ -78,7 +81,7 @@ public class ViolationSourceCodeTree { } } - + public void extract(String type) { ITree rootTree = new GumTreeGenerator().generateITreeForJavaFile(file, GumTreeType.EXP_JDT); @@ -296,7 +299,9 @@ public class ViolationSourceCodeTree { int endPosition = startPosition + tree.getLength(); int endLine = cUnit.getLineNumber(endPosition); - if (endLine < violationStartLine) continue; + if (endLine < violationStartLine) { + continue; + } if (endLine < violationEndLine) { if ("NM_SAME_SIMPLE_NAME_AS_INTERFACE".equals(type) || "NM_SAME_SIMPLE_NAME_AS_SUPERCLASS".equals(type) || "NM_CLASS_NAMING_CONVENTION".equals(type) @@ -347,7 +352,9 @@ public class ViolationSourceCodeTree { break; } - if (tree.getType() == 31 || tree.getType() == 23) { // MethodDeclaration, FieldDeclaration + // Class Name(super class etc.), Field, initializer, EnumDeclaration, EnumConstantDeclaration, Method Name, method body, + if (tree.getType() == 31 || tree.getType() == 23 || tree.getType() == 28 || tree.getType() == 71 || tree.getType() == 72) { + // MethodDeclaration, FieldDeclaration, Initializer, EnumDeclaration, EnumConstantDeclaration this.violationFinalStartLine = startLine; this.violationFinalEndLine = endLine; break; @@ -362,8 +369,19 @@ public class ViolationSourceCodeTree { for (int i = 0, size = children.size(); i < size; i ++) { ITree child = children.get(i); int type = child.getType(); - if (type == 8) { - return child.getPos() - 1; + if (type == 6 || type == 10 || type == 12 || type == 17 || type == 18 || type == 19 || type == 21 || type == 8// Block, EmptyStatement + || type == 24 || type == 25 || type == 30 || type == 41 || type == 46 || type == 49 || type == 50 + || type == 51 || type == 53 || type == 54 || type == 56 || type == 60 || type == 61 || type == 70) { + //AssertStatement, BreakStatement, CatchClause, ConstructorInvocation, ContinueStatement, DoStatement + // ExpressionStatement, ForStatement, IfStatement, LabeledStatement, ReturnStatement, SuperConstructorInvocation + // SwitchCase, SwitchStatement, SynchronizedStatement, ThrowStatement, TryStatement + // TypeDeclarationStatement, VariableDeclarationStatement, WhileStatement, EnhancedForStatement + if ( i > 0) { + child = children.get(i - 1); + return child.getPos() + child.getLength(); + } else { + return child.getPos() - 1; + } } } return 0; @@ -417,4 +435,261 @@ public class ViolationSourceCodeTree { return parent; } + public void locateParentNode() { + ITree rootTree = new GumTreeGenerator().generateITreeForJavaFile(file, GumTreeType.EXP_JDT); + + List trees = rootTree.getChildren(); + for (ITree tree : trees) { + int startPosition = tree.getPos(); + int startLine = cUnit.getLineNumber(startPosition); + if (startLine > violationEndLine) { + break; + } + + int endPosition = startPosition + tree.getLength(); + int endLine = cUnit.getLineNumber(endPosition - 1); + if (endLine < violationStartLine) continue; + + locateParentNode(tree.getChildren()); + } + +// if (this.violationFinalStartLine == 0) { +// FileHelper.outputToFile("logs/testV1.txt", type + " : " + this.file.getName() + " : " + this.violationStartLine + " : " + this.violationEndLine + "\n", true); +// } + } + + private void locateParentNode(List trees) { + for (ITree tree : trees) { + int startPosition = tree.getPos(); + int startLine = cUnit.getLineNumber(startPosition); + if (startLine > violationEndLine) { + break; + } + + int endPosition = startPosition + tree.getLength(); + int endLine = cUnit.getLineNumber(endPosition); + if (endLine < violationStartLine) { + continue; + } + + if (endLine < violationEndLine) { + if (this.violationFinalStartLine == 0) { + this.violationFinalStartLine = -1; + } + break; + } + + int type = tree.getType(); + // Class Name(super class etc.), Method Name, + if (type == 31) { //MethodDeclaration, TypeDeclaration + this.violationFinalStartLine = type; + locateParentNode(tree.getChildren()); + break; + } else if (type == 23 || type == 28 || type == 71 || type == 72 + || type == 81 || type == 82|| type == 1) { + // FieldDeclaration, Initializer, EnumDeclaration, EnumConstantDeclaration, + // AnnotationTypeDeclaration, AnnotationTypeMemberDeclaration, AnonymousClassDeclaration + this.violationFinalStartLine = tree.getType(); + break; + } else { + locateParentNode(tree.getChildren()); + } + } + } + + /** + * extract source code of violations. + */ + public void extract2() { + ITree rootTree = new GumTreeGenerator().generateITreeForJavaFile(file, GumTreeType.EXP_JDT); + + List trees = rootTree.getChildren(); + for (ITree tree : trees) { + int startPosition = tree.getPos(); + int startLine = cUnit.getLineNumber(startPosition); + if (startLine > violationEndLine) { + break; + } + + int endPosition = startPosition + tree.getLength(); + int endLine = cUnit.getLineNumber(endPosition - 1); + if (endLine < violationStartLine) continue; + + matchTrees2(tree.getChildren()); + } + + int size = matchedTrees.size(); + if (size > 0) { + if (this.violationFinalStartLine != 0) { + this.violationFinalStartLine = cUnit.getLineNumber(this.matchedTrees.get(0).getPos()); + ITree lastTree = matchedTrees.get(size - 1); + this.violationFinalEndLine = cUnit.getLineNumber(lastTree.getPos() + lastTree.getLength()); + } + } else { + System.err.println(this.file.getName() + "===" + this.violationStartLine + ":" + this.violationEndLine); + } + + } + + private void matchTrees2(List trees) { + for (ITree tree : trees) { + int startPosition = tree.getPos(); + int startLine = cUnit.getLineNumber(startPosition); + if (startLine > violationEndLine) { + break; + } + + int endPosition = startPosition + tree.getLength(); + int endLine = cUnit.getLineNumber(endPosition); + if (endLine < violationStartLine) continue; + + if (endLine == violationEndLine) { + if (tree.getType() == 31) { // MethodDeclaration + if (startLine == violationStartLine) { + + } else { + matchTrees(tree.getChildren()); + } + } else if (isStatement(tree)) { + addToMatchedTrees(tree); + } else { + ITree parent = getParentStatement2(tree); + if (parent == null) { + if (tree.getType() == 8) { // 8: Block + matchTrees(tree.getChildren()); + } + continue; + } else if (parent.getType() == 31) { // method name + if (startLine == violationStartLine || endLine < violationEndLine) { + int finalEndPosition = getMethodDeclarationWithBody(parent); + addToMatchedTrees(parent); + this.violationFinalStartLine = cUnit.getLineNumber(parent.getPos()); + this.violationFinalEndLine = cUnit.getLineNumber(finalEndPosition); + } + break; + } else if (parent.getType() == 55) { // class name + if (startLine == violationStartLine || endLine < violationEndLine) { + int finalEndPosition = getClassDecalrationWithBody(parent); + addToMatchedTrees(parent); + this.violationFinalStartLine = cUnit.getLineNumber(parent.getPos()); + this.violationFinalEndLine = cUnit.getLineNumber(finalEndPosition); + } + break; + } + addToMatchedTrees(parent); + } + continue; + } + + if (startLine >= violationStartLine) { + if (isStatement(tree)) { + addToMatchedTrees(tree); + } else { + ITree parent = getParentStatement2(tree); + if (parent == null) { + if (tree.getType() == 8) { + matchTrees(tree.getChildren()); + } + continue; + } else if (parent.getType() == 31) { // method name + if (startLine == violationStartLine || endLine < violationEndLine) { + int finalEndPosition = getMethodDeclarationWithBody(parent); + addToMatchedTrees(parent); + this.violationFinalStartLine = cUnit.getLineNumber(parent.getPos()); + this.violationFinalEndLine = cUnit.getLineNumber(finalEndPosition); + } + break; + } else if (parent.getType() == 55) { // class name + if (startLine == violationStartLine || endLine < violationEndLine) { + int finalEndPosition = getClassDecalrationWithBody(parent); + addToMatchedTrees(parent); + this.violationFinalStartLine = cUnit.getLineNumber(parent.getPos()); + this.violationFinalEndLine = cUnit.getLineNumber(finalEndPosition); + } + break; + } + addToMatchedTrees(parent); + } + } else { + matchTrees(tree.getChildren()); + } + } + } + + private int getMethodDeclarationWithBody(ITree tree) { + List children = tree.getChildren(); + List newChildren = new ArrayList<>(); + for (int i = 0, size = children.size(); i < size; i ++) { + ITree child = children.get(i); + int type = child.getType(); + if (type == 6 || type == 10 || type == 12 || type == 17 || type == 18 || type == 19 || type == 21 || type == 8// Block, EmptyStatement + || type == 24 || type == 25 || type == 30 || type == 41 || type == 46 || type == 49 || type == 50 + || type == 51 || type == 53 || type == 54 || type == 56 || type == 60 || type == 61 || type == 70) { + //AssertStatement, BreakStatement, CatchClause, ConstructorInvocation, ContinueStatement, DoStatement + // ExpressionStatement, ForStatement, IfStatement, LabeledStatement, ReturnStatement, SuperConstructorInvocation + // SwitchCase, SwitchStatement, SynchronizedStatement, ThrowStatement, TryStatement + // TypeDeclarationStatement, VariableDeclarationStatement, WhileStatement, EnhancedForStatement + if (i > 0) { + child = children.get(i - 1); + return child.getPos() + child.getLength() + 1; + } else { + return child.getPos() - 1; + } + } else { + newChildren.add(child); + } + } + tree.setChildren(newChildren); + return 0; + } + + private int getClassDecalrationWithBody(ITree tree) { + List children = tree.getChildren(); + List newChildren = new ArrayList<>(); + for (int i = 0, size = children.size(); i < size; i ++) { + ITree child = children.get(i); + int type = child.getType(); + // Modifier, NormalAnnotation, MarkerAnnotation, SingleMemberAnnotation + if (type != 83 && type != 77 && type != 78 && type != 79 + && type != 5 && type != 39 && type != 43 && type != 74 && type != 75 + && type != 76 && type != 84 && type != 87 && type != 88) { + // ArrayType, PrimitiveType, SimpleType, ParameterizedType, + // QualifiedType, WildcardType, UnionType, IntersectionType, NameQualifiedType + if (i > 0) { + child = children.get(i - 1); + return child.getPos() + child.getLength() + 1; + } else { + return child.getPos() - 1; + } + } else { + newChildren.add(child); + } + } + tree.setChildren(newChildren); + return 0; + } + + + private ITree getParentStatement2(ITree tree) { + ITree parent = tree; + do { + parent = parent.getParent(); + if (parent == null) { + return null; + } + + int type = parent.getType(); + if (type == 31 || type == 55) { + // MethodDeclaration + // TypeDeclaration + return parent; + } else if (type == 1 || type == 71) { + // AnonymousClassDeclaration + // EnumDeclaration + return null; + } + } while (!isStatement(parent)); + + return parent; + } } diff --git a/src/main/java/edu/lu/uni/serval/violation/parse/TestViolationParser.java b/src/main/java/edu/lu/uni/serval/violation/parse/TestViolationParser.java index c021c88..3686535 100644 --- a/src/main/java/edu/lu/uni/serval/violation/parse/TestViolationParser.java +++ b/src/main/java/edu/lu/uni/serval/violation/parse/TestViolationParser.java @@ -10,7 +10,7 @@ import edu.lu.uni.serval.utils.FileHelper; public class TestViolationParser { - private static final String REPO_PATH = "/Volumes/MacBook/repos/"; + private static final String REPO_PATH = "../../repos/";//"/Volumes/MacBook/repos/"; public static void main(String[] args) throws IOException { List repositoriesList = new ArrayList<>(); @@ -31,8 +31,6 @@ public class TestViolationParser { String unfixedViolations = "../FPM_Violations/unFixedInstances/"; String unfixedFilesPath = Configuration.GUM_TREE_INPUT + "UnfixedViolations/"; String unfixedPositionsFilePath = Configuration.GUM_TREE_INPUT + "UnFV_positions/"; - FileHelper.createDirectory(unfixedFilesPath); - FileHelper.createDirectory(unfixedPositionsFilePath); List unfixedAlarmFiles = FileHelper.getAllFilesInCurrentDiectory(unfixedViolations, ".list"); for (File file : unfixedAlarmFiles) { String fileName = FileHelper.getFileNameWithoutExtension(file);