From 21d55e82daa1274652a3bc7ff0a8743153e2b307 Mon Sep 17 00:00:00 2001 From: Kui LIU Date: Sat, 5 Aug 2017 11:25:36 +0200 Subject: [PATCH] Update --- .gitignore | 4 + pom.xml | 4 +- .../FixPatternParser/RunnableParser.java | 2 +- .../patch/CommitPatchHunkParser.java | 4 +- .../CommitPatchSingleStatementParser.java | 37 ++++------ .../violations/FixedViolationHunkParser.java | 39 ++++++---- .../violations/FixedViolationParser.java | 10 ++- .../FixedViolationSingleStatementParser.java | 43 +++++------ .../MultipleThreadsParser/AkkaParser.java | 2 +- .../MultipleThreadsParser/MessageFile.java | 9 +++ .../uni/serval/diffentry/DiffEntryReader.java | 74 ++++++++++++++++++- 11 files changed, 156 insertions(+), 72 deletions(-) diff --git a/.gitignore b/.gitignore index 61a782f..d6572fc 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,10 @@ OUTPUT/ *.pdf Dataset/ GumTreeInput/ +GumTreeInput1/ +GumTreeOutput1/ +GumTreeResults/ + *TestParser.java src/main/java/edu/lu/uni/serval/FixPatternParser/TestParser.java # Package Files # diff --git a/pom.xml b/pom.xml index 27101a1..412a205 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ - + edu.lu.uni diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java index 82f66fc..850a5a2 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java @@ -8,7 +8,7 @@ public class RunnableParser implements Runnable { private File revFile; private File diffentryFile; private Parser parser; - + public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) { this.prevFile = prevFile; this.revFile = revFile; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchHunkParser.java index ebe4284..4eb7a77 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchHunkParser.java @@ -36,9 +36,9 @@ public class CommitPatchHunkParser extends CommitPatchParser { List diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile); //Filter out the modify actions, which are not in the DiffEntry hunks. HunkActionFilter hunkFilter = new HunkActionFilter(); - List allHunkFixPatternss = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, actionSets, revFile, prevFile); + List allHunkFixPatterns = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, actionSets, revFile, prevFile); - for (HunkFixPattern hunkFixPattern : allHunkFixPatternss) { + for (HunkFixPattern hunkFixPattern : allHunkFixPatterns) { // Range of buggy source code int startLine = 0; int endLine = 0; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchSingleStatementParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchSingleStatementParser.java index 5af42b4..795d628 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchSingleStatementParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/patch/CommitPatchSingleStatementParser.java @@ -37,6 +37,12 @@ public class CommitPatchSingleStatementParser extends CommitPatchParser { if (actionSets.size() > 0) { // DiffEntry Hunks: filter out big hunks. List diffentryHunks = new DiffEntryReader().readHunks(diffEntryFile); + CUCreator cuCreator = new CUCreator(); + CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); + CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); + if (prevUnit == null || revUnit == null) { + return; + } for (HierarchicalActionSet actionSet : actionSets) { // position of buggy statements int startPosition = 0; @@ -80,31 +86,21 @@ public class CommitPatchSingleStatementParser extends CommitPatchParser { endPosition2 = startPosition2 + newNode.getLength(); } } - } else {// DEL actions and MOV actions: we don't need these - // actions, as for now. + } else {// DEL actions and MOV actions: we don't need these actions, as for now. continue; } if (startPosition == 0 || startPosition2 == 0) { continue; } - CUCreator cuCreator = new CUCreator(); - CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); - CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); - if (prevUnit == null || revUnit == null) { - continue; - } - // Get line numbers. int startLine = prevUnit.getLineNumber(startPosition); int endLine = prevUnit.getLineNumber(endPosition); int startLine2 = revUnit.getLineNumber(startPosition2); int endLine2 = revUnit.getLineNumber(endPosition2); - if (endLine - startLine >= Configuration.HUNK_SIZE - 2 - || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2) + if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2) continue; - // Filter out the modify actions, which are not in the DiffEntry - // hunks. + // Filter out the modify actions, which are not in the DiffEntry hunks. DiffEntryHunk hunk = matchHunk(startLine, endLine, startLine2, endLine2, actionStr, diffentryHunks); if (hunk == null) { continue; @@ -117,8 +113,7 @@ public class CommitPatchSingleStatementParser extends CommitPatchParser { SimplifyTree abstractIdentifier = new SimplifyTree(); abstractIdentifier.abstractTree(actionSet); SimpleTree simpleTree = actionSet.getSimpleTree(); - if (simpleTree == null) { // Failed to get the simple tree for - // INS actions. + if (simpleTree == null) { // Failed to get the simple tree for INS actions. continue; } @@ -142,12 +137,12 @@ public class CommitPatchSingleStatementParser extends CommitPatchParser { this.patchesSourceCode += Configuration.PATCH_SIGNAL + "\n" + patchSourceCode + "\n"; this.sizes += size + "\n"; this.astEditScripts += astEditScripts + "\n"; - // 2. source code: raw tokens - String rawTokenEditScripts = getRawTokenEditScripts(actionSet); - // 3. abstract identifiers: - String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet); - // 4. semi-source code: - String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet); +// // 2. source code: raw tokens +// String rawTokenEditScripts = getRawTokenEditScripts(actionSet); +// // 3. abstract identifiers: +// String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet); +// // 4. semi-source code: +// String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet); // this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + // simpleTree.toString() + "\n"; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java index 4f7ed37..b497088 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java @@ -12,7 +12,6 @@ import edu.lu.uni.serval.FixPatternParser.Tokenizer; import edu.lu.uni.serval.config.Configuration; import edu.lu.uni.serval.diffentry.DiffEntryHunk; import edu.lu.uni.serval.diffentry.DiffEntryReader; -import edu.lu.uni.serval.gumtree.regroup.ActionFilter; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import edu.lu.uni.serval.gumtree.regroup.HunkActionFilter; import edu.lu.uni.serval.gumtree.regroup.HunkFixPattern; @@ -21,7 +20,7 @@ import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; import edu.lu.uni.serval.utils.MapSorter; /** - * Parse fix patterns with GumTree. + * Parse fix violations with GumTree. * * @author kui.liu * @@ -29,36 +28,44 @@ import edu.lu.uni.serval.utils.MapSorter; public class FixedViolationHunkParser extends FixedViolationParser { @Override - public void parseFixPatterns(File prevFile, File revFile, File positionsFile) { + public void parseFixPatterns(File prevFile, File revFile, File diffentryFile) { // GumTree results List actionSets = parseChangedSourceCodeWithGumTree(prevFile, revFile); if (actionSets.size() > 0) { - Map positions = readPositions(positionsFile); + Map positions = readPositions(); if (positions.size() > 1) { MapSorter sorter = new MapSorter<>(); positions = sorter.sortByKeyAscending(positions); } + List diffentryHunks1 = new DiffEntryReader().readHunks(diffentryFile); + int index = 0; + int hunkListSize = diffentryHunks1.size(); +// Map diffentryHunks = new HashMap<>(); + // Select hunks by positions of violations. + List diffentryHunks = new ArrayList<>(); for (Map.Entry entry : positions.entrySet()) { - // only statements - for (HierarchicalActionSet actionSet : actionSets) { - String astNodeType = actionSet.getAstNodeType(); - if (astNodeType.endsWith("Statement") || "FieldDeclaration".equals(astNodeType)) { - - } + int startRange = entry.getKey(); + int endRange = entry.getValue(); + for (; index < hunkListSize; index ++) { + DiffEntryHunk hunk = diffentryHunks1.get(index); + int startLine = hunk.getBugLineStartNum(); + int range = hunk.getBugRange(); + if (startRange > startLine + range) continue; + if (endRange < startLine) break; + // startRange and endRange +// diffentryHunks.put(startRange, hunk); + diffentryHunks.add(hunk); } } - ActionFilter filter = new ActionFilter(); - // DiffEntry size: filter out big hunks. - List diffentryHunks = new DiffEntryReader().readHunks(positionsFile); //Filter out the modify actions, which are not in the DiffEntry hunks. HunkActionFilter hunkFilter = new HunkActionFilter(); - List allHunkFixPatternss = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, actionSets, revFile, prevFile); - - for (HunkFixPattern hunkFixPattern : allHunkFixPatternss) { + List allHunkFixPatterns = hunkFilter.filterActionsByDiffEntryHunk2(diffentryHunks, actionSets, revFile, prevFile); + + for (HunkFixPattern hunkFixPattern : allHunkFixPatterns) { // Range of buggy source code int startLine = 0; int endLine = 0; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java index 5a63237..56c98f9 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java @@ -18,8 +18,14 @@ import edu.lu.uni.serval.utils.FileHelper; */ public class FixedViolationParser extends Parser { + File positionFile = null; + + public void setPositionFile(File positionFile) { + this.positionFile = positionFile; + } + @Override - public void parseFixPatterns(File prevFile, File revFile, File positionFile) { + public void parseFixPatterns(File prevFile, File revFile, File diffentryFile) { } protected boolean inPositions(int startLine, int endLine, Map positions) { @@ -31,7 +37,7 @@ public class FixedViolationParser extends Parser { return false; } - protected Map readPositions(File positionFile) { + protected Map readPositions() { Map positions = new HashMap<>(); String fileContent = FileHelper.readFile(positionFile); BufferedReader reader = null; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java index 20b3d11..c4d6fa5 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationSingleStatementParser.java @@ -19,7 +19,7 @@ import edu.lu.uni.serval.gumtree.regroup.SimpleTree; import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; /** - * Parse fix patterns with GumTree. + * Parse fixed violations with GumTree. * * @author kui.liu * @@ -27,13 +27,20 @@ import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; public class FixedViolationSingleStatementParser extends FixedViolationParser { @Override - public void parseFixPatterns(File prevFile, File revFile, File positionFile) { + public void parseFixPatterns(File prevFile, File revFile, File diffentryFile) { // GumTree results List actionSets = parseChangedSourceCodeWithGumTree(prevFile, revFile); if (actionSets.size() > 0) { - // DiffEntry Hunks: filter out big hunks. - Map positions = readPositions(positionFile); + CUCreator cuCreator = new CUCreator(); + CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); + CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); + if (prevUnit == null || revUnit == null) { + return; + } + + // Read the positions of checked violations + Map positions = readPositions(); for (HierarchicalActionSet actionSet : actionSets) { // position of buggy statements int startPosition = 0; @@ -76,29 +83,13 @@ public class FixedViolationSingleStatementParser extends FixedViolationParser { endPosition2 = startPosition2 + newNode.getLength(); } } - } else if (actionStr.startsWith("MOV")) {// DEL actions and MOV actions: we don't need these actions, as for now. - continue; -// startPosition = actionSet.getStartPosition(); -// endPosition = startPosition + actionSet.getLength(); -// ITree node = actionSet.getNode().getParent(); -// startPosition2 = node.getPos(); -// endPosition2 = startPosition2 + node.getLength(); - } else { - startPosition = actionSet.getStartPosition(); - endPosition = startPosition + actionSet.getLength(); + } else {// DEL actions and MOV actions: we don't need these actions, as for now. continue; } if (startPosition == 0 || startPosition2 == 0) { continue; } - CUCreator cuCreator = new CUCreator(); - CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); - CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); - if (prevUnit == null || revUnit == null) { - continue; - } - // Get line numbers. int startLine = prevUnit.getLineNumber(startPosition); int endLine = prevUnit.getLineNumber(endPosition); @@ -138,11 +129,11 @@ public class FixedViolationSingleStatementParser extends FixedViolationParser { this.sizes += size + "\n"; this.astEditScripts += astEditScripts + "\n"; // 2. source code: raw tokens - String rawTokenEditScripts = getRawTokenEditScripts(actionSet); - // 3. abstract identifiers: - String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet); - // 4. semi-source code: - String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet); +// String rawTokenEditScripts = getRawTokenEditScripts(actionSet); +// // 3. abstract identifiers: +// String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet); +// // 4. semi-source code: +// String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet); // this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n"; this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n"; diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/AkkaParser.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/AkkaParser.java index cebec75..aec1eb3 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/AkkaParser.java +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/AkkaParser.java @@ -55,10 +55,10 @@ public class AkkaParser { List msgFiles = new ArrayList<>(); for (File file : files) { + if (!file.isDirectory()) continue; String projectFolder = file.getPath(); File revFileFolder = new File(projectFolder + "/revFiles/");// revised file folder File[] revFiles = revFileFolder.listFiles(); - for (File revFile : revFiles) { if (revFile.getName().endsWith(".java")) { File prevFile = new File(projectFolder + "/prevFiles/prev_" + revFile.getName());// previous file diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/MessageFile.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/MessageFile.java index 121eeb3..de224ef 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/MessageFile.java +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/MessageFile.java @@ -7,6 +7,7 @@ public class MessageFile { private File revFile; private File prevFile; private File diffEntryFile; + private File positionFile; public MessageFile(File revFile, File prevFile, File diffEntryFile) { super(); @@ -26,5 +27,13 @@ public class MessageFile { public File getDiffEntryFile() { return diffEntryFile; } + + public File getPositionFile() { + return positionFile; + } + + public void setPositionFile(File positionFile) { + this.positionFile = positionFile; + } } diff --git a/src/main/java/edu/lu/uni/serval/diffentry/DiffEntryReader.java b/src/main/java/edu/lu/uni/serval/diffentry/DiffEntryReader.java index b7f9881..4cc2802 100644 --- a/src/main/java/edu/lu/uni/serval/diffentry/DiffEntryReader.java +++ b/src/main/java/edu/lu/uni/serval/diffentry/DiffEntryReader.java @@ -53,7 +53,7 @@ public class DiffEntryReader { } hunk.append(line + "\n"); } - + if ((range < 7 && range2 < 7) || range == 0 || range2 == 0) { // filter out big hunks DiffEntryHunk diffEntryHunk = new DiffEntryHunk(startLine, startLine2, range, range2); diffEntryHunk.setHunk(hunk.toString()); @@ -77,4 +77,76 @@ public class DiffEntryReader { return diffentryHunks; } + /** + * Read all hunks without considering their sizes. + * + * @param diffentryFile + * @return + */ + public List readHunks2(File diffentryFile) { + List diffentryHunks = new ArrayList<>(); + String content = FileHelper.readFile(diffentryFile); + BufferedReader reader = null; + try { + reader = new BufferedReader(new StringReader(content)); + String line = null; + int startLine = 0; + int range = 0; + int startLine2 = 0; + int range2 = 0; + StringBuilder hunk = new StringBuilder(); + + while ((line = reader.readLine()) != null) { + if (RegExp.filterSignal(line.trim())) { + if (hunk.length() > 0) { + if (startLine > 0) { + DiffEntryHunk diffEntryHunk = new DiffEntryHunk(startLine, startLine2, range, range2); + diffEntryHunk.setHunk(hunk.toString()); + diffentryHunks.add(diffEntryHunk); + } + hunk.setLength(0); + } + int plusIndex = line.indexOf("+"); + String lineNum = line.substring(4, plusIndex); + String[] nums = lineNum.split(","); + startLine = Integer.parseInt(nums[0].trim()); + if (nums.length == 2) { + range = Integer.parseInt(nums[1].trim()); + } + + String lineNum2 = line.substring(plusIndex) .trim(); + lineNum2 = lineNum2.substring(1, lineNum2.length() - 2); + String[] nums2 = lineNum2.split(","); + startLine2 = Integer.parseInt(nums2[0].trim()); + if (nums2.length == 2) { + range2 = Integer.parseInt(nums2[1].trim()); + } + continue; + } + hunk.append(line + "\n"); + } + + if (startLine > 0) { + DiffEntryHunk diffEntryHunk = new DiffEntryHunk(startLine, startLine2, range, range2); + diffEntryHunk.setHunk(hunk.toString()); + diffentryHunks.add(diffEntryHunk); + } + hunk.setLength(0); + + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + if (reader != null) { + reader.close(); + reader = null; + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + return diffentryHunks; + } + }