From f2117d4483138cf85b523883701c48a0cb6990d7 Mon Sep 17 00:00:00 2001 From: Kui LIU Date: Thu, 3 Aug 2017 15:43:00 +0200 Subject: [PATCH] Implement the parsing of violations. --- .../violations/FixedViolationParser.java | 390 ++++++++++++++++++ .../edu/lu/uni/serval/violation/Alarm.java | 66 +++ .../lu/uni/serval/violation/Violation.java | 25 ++ .../serval/violation/parse/AlarmsReader.java | 75 ++++ .../violation/parse/ViolationParser.java | 112 +++++ 5 files changed, 668 insertions(+) create mode 100644 src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java create mode 100644 src/main/java/edu/lu/uni/serval/violation/Alarm.java create mode 100644 src/main/java/edu/lu/uni/serval/violation/Violation.java create mode 100644 src/main/java/edu/lu/uni/serval/violation/parse/AlarmsReader.java create mode 100644 src/main/java/edu/lu/uni/serval/violation/parse/ViolationParser.java diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java new file mode 100644 index 0000000..e34c453 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java @@ -0,0 +1,390 @@ +package edu.lu.uni.serval.FixPatternParser.violations; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.eclipse.jdt.core.dom.CompilationUnit; + +import com.github.gumtreediff.actions.model.Action; +import com.github.gumtreediff.actions.model.Move; +import com.github.gumtreediff.actions.model.Update; +import com.github.gumtreediff.tree.ITree; + +import edu.lu.uni.serval.FixPattern.utils.Checker; +import edu.lu.uni.serval.FixPatternParser.CUCreator; +import edu.lu.uni.serval.FixPatternParser.Tokenizer; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.gumtree.GumTreeComparer; +import edu.lu.uni.serval.gumtree.regroup.ActionFilter; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; +import edu.lu.uni.serval.gumtree.regroup.SimpleTree; +import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; +import edu.lu.uni.serval.utils.FileHelper; + +/** + * Parse fix patterns with GumTree. + * + * @author kui.liu + * + */ +public class FixedViolationParser { + + private String astEditScripts = ""; // it will be used for fix patterns mining. + private String patchesSourceCode = ""; // testing + private String buggyTrees = ""; // Compute similarity for bug localization. + private String sizes = ""; // fix patterns' selection before mining. + private String tokensOfSourceCode = ""; // Compute similarity for bug localization. + private String originalTree = ""; // Guide of generating patches. + private String actionSets = ""; // Guide of generating patches. + + public void parseFixPatterns(File prevFile, File revFile, File positionFile) { + // GumTree results + List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); + + if (gumTreeResults != null && gumTreeResults.size() > 0) { + List allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults); + // Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part. + // TODO: variable effects range, sub-actions are these kinds of modification? + List actionSets = new ActionFilter().filterOutUselessActions(allActionSets); + + if (actionSets.size() > 0) { + // DiffEntry Hunks: filter out big hunks. + Map positions = readPositions(positionFile); + for (HierarchicalActionSet actionSet : actionSets) { + // position of buggy statements + int startPosition = 0; + int endPosition = 0; + // position of fixed statements + int startPosition2 = 0; + int endPosition2 = 0; + + String actionStr = actionSet.getActionString(); + String astNodeType = actionSet.getAstNodeType(); + if (actionStr.startsWith("INS")) { + startPosition2 = actionSet.getStartPosition(); + endPosition2 = startPosition2 + actionSet.getLength(); + List firstAndLastMov = getFirstAndLastMoveAction(actionSet); + if (firstAndLastMov != null) { + startPosition = firstAndLastMov.get(0).getNode().getPos(); + ITree lastTree = firstAndLastMov.get(1).getNode(); + endPosition = lastTree.getPos() + lastTree.getLength(); + } else { // Ignore the pure insert actions without any move actions. + continue; + } + } else if (actionStr.startsWith("UPD")) { + startPosition = actionSet.getStartPosition(); + endPosition = startPosition + actionSet.getLength(); + Update update = (Update) actionSet.getAction(); + ITree newNode = update.getNewNode(); + startPosition2 = newNode.getPos(); + endPosition2 = startPosition2 + newNode.getLength(); + + if (Checker.containsBodyBlock(astNodeType)) { + List children = update.getNode().getChildren(); + endPosition = getEndPosition(children); + List newChildren = newNode.getChildren(); + endPosition2 = getEndPosition(newChildren); + + if (endPosition == 0) { + endPosition = startPosition + actionSet.getLength(); + } + if (endPosition2 == 0) { + endPosition2 = startPosition2 + newNode.getLength(); + } + } + } else {// DEL actions and MOV actions: we don't need these actions, as for now. + continue; + } + if (startPosition == 0 || startPosition2 == 0) { + continue; + } + + CUCreator cuCreator = new CUCreator(); + CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); + CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); + if (prevUnit == null || revUnit == null) { + continue; + } + + // Get line numbers. + int startLine = prevUnit.getLineNumber(startPosition); + int endLine = prevUnit.getLineNumber(endPosition); + int startLine2 = revUnit.getLineNumber(startPosition2); + int endLine2 = revUnit.getLineNumber(endPosition2); + + if (!inPositions(startLine, endLine, positions)) { + continue; + } +// if (endLine - startLine >= Configuration.HUNK_SIZE - 2 || endLine2 - startLine2 >= Configuration.HUNK_SIZE - 2 ) continue; + + /* + * Convert the ITree of buggy code to a simple tree. + * It will be used to compute the similarity. + */ + SimplifyTree abstractIdentifier = new SimplifyTree(); + abstractIdentifier.abstractTree(actionSet); + SimpleTree simpleTree = actionSet.getSimpleTree(); + if (simpleTree == null) { // Failed to get the simple tree for INS actions. + continue; + } + + /** + * Select edit scripts for deep learning. + * Edit scripts will be used to mine common fix patterns. + */ + // 1. First level: AST node type. + String astEditScripts = getASTEditScripts(actionSet); + int size = astEditScripts.split(" ").length; + if (size == 1) { + continue; + } + + // Source Code of patches. + String patchSourceCode = getPatchSourceCode(prevFile, revFile, startLine, endLine, startLine2, endLine2); + this.patchesSourceCode += Configuration.PATCH_SIGNAL + "\n" + revFile.getName() + "\n" + patchSourceCode + "\n"; + this.sizes += size + "\n"; + this.astEditScripts += astEditScripts + "\n"; + // 2. source code: raw tokens + String rawTokenEditScripts = getRawTokenEditScripts(actionSet); + // 3. abstract identifiers: + String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet); + // 4. semi-source code: + String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet); + +// this.buggyTrees += Configuration.BUGGY_TREE_TOKEN + "\n" + simpleTree.toString() + "\n"; + this.tokensOfSourceCode += Tokenizer.getTokensDeepFirst(simpleTree).trim() + "\n"; +// this.actionSets += Configuration.BUGGY_TREE_TOKEN + "\n" + readActionSet(actionSet, "") + "\n"; +// this.originalTree += Configuration.BUGGY_TREE_TOKEN + "\n" + actionSet.getOriginalTree().toString() + "\n"; + } + actionSets.clear(); + } + allActionSets.clear(); + gumTreeResults.clear(); + } + } + + private boolean inPositions(int startLine, int endLine, Map positions) { + for (Map.Entry entry : positions.entrySet()) { + int startPosi = entry.getKey(); + int endPosi = entry.getValue(); + if (endLine >= startPosi && startLine <= endPosi) return true; + } + return false; + } + + private Map readPositions(File positionFile) { + Map positions = new HashMap<>(); + String fileContent = FileHelper.readFile(positionFile); + BufferedReader reader = null; + reader = new BufferedReader(new StringReader(fileContent)); + String line = null; + try { + while ((line = reader.readLine()) != null) { + String[] positionStr = line.split(":"); + int startLine = Integer.parseInt(positionStr[0]); + int endLine = Integer.parseInt(positionStr[1]); + positions.put(startLine, endLine); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return positions; + } + + private List getFirstAndLastMoveAction(HierarchicalActionSet gumTreeResult) { + List firstAndLastMoveActions = new ArrayList<>(); + List actions = gumTreeResult.getSubActions(); + if (actions.size() == 0) { + return null; + } + Move firstMoveAction = null; + Move lastMoveAction = null; + while (actions.size() > 0) { + List subActions = new ArrayList<>(); + for (HierarchicalActionSet action : actions) { + subActions.addAll(action.getSubActions()); + if (action.toString().startsWith("MOV")) { + if (firstMoveAction == null) { + firstMoveAction = (Move) action.getAction(); + lastMoveAction = (Move) action.getAction(); + } else { + int startPosition = action.getStartPosition(); + int length = action.getLength(); + int startPositionFirst = firstMoveAction.getPosition(); + int startPositionLast = lastMoveAction.getPosition(); + int lengthLast = lastMoveAction.getNode().getLength(); + if (startPosition < startPositionFirst || (startPosition == startPositionFirst && length > firstMoveAction.getLength())) { + firstMoveAction = (Move) action.getAction(); + } + if ((startPosition + length) > (startPositionLast + lengthLast)) { + lastMoveAction = (Move) action.getAction(); + } + } + } + } + + actions.clear(); + actions.addAll(subActions); + } + if (firstMoveAction == null) { + return null; + } + firstAndLastMoveActions.add(firstMoveAction); + firstAndLastMoveActions.add(lastMoveAction); + return firstAndLastMoveActions; + } + + private String readActionSet(HierarchicalActionSet actionSet, String line) { + String str = line + actionSet.getActionString() + "\n"; + List subActions = actionSet.getSubActions(); + for (HierarchicalActionSet subAction : subActions) { + str += readActionSet(subAction, line + "---"); + } + return str; + } + + private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) { + // TODO Auto-generated method stub + return null; + } + + private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) { + // TODO Auto-generated method stub + return null; + } + + private String getRawTokenEditScripts(HierarchicalActionSet actionSet) { + // TODO Auto-generated method stub + return null; + } + + private int getEndPosition(List children) { + int endPosition = 0; + for (ITree child : children) { + if (child.getLabel().endsWith("Body")) { + endPosition = child.getPos() - 1; + break; + } + } + return endPosition; + } + + private String getPatchSourceCode(File prevFile, File revFile, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) { + String buggyStatements = readSourceCode(prevFile, startLineNum, endLineNum, "-"); + String fixedStatements = readSourceCode(revFile, startLineNum2, endLineNum2, "+"); + return buggyStatements + fixedStatements; + } + + private String readSourceCode(File file, int startLineNum, int endLineNum, String type) { + String sourceCode = ""; + String fileContent = FileHelper.readFile(file); + BufferedReader reader = null; + try { + reader = new BufferedReader(new StringReader(fileContent)); + String line = null; + int lineIndex = 0; + while ((line = reader.readLine()) != null) { + lineIndex ++; + if (lineIndex >= startLineNum && lineIndex <= endLineNum) { + sourceCode += type + line + "\n"; + } + if (lineIndex == endLineNum) break; + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return sourceCode; + } + + /** + * Get the AST node based edit script of patches in terms of breadth first. + * + * @param actionSet + * @return + */ + private String getASTEditScripts(HierarchicalActionSet actionSet) { + String editScript = ""; + + List actionSets = new ArrayList<>(); + actionSets.add(actionSet); + while (actionSets.size() != 0) { + List subSets = new ArrayList<>(); + for (HierarchicalActionSet set : actionSets) { + subSets.addAll(set.getSubActions()); + String actionStr = set.getActionString(); + int index = actionStr.indexOf("@@"); + String singleEdit = actionStr.substring(0, index).replace(" ", ""); + + if (singleEdit.endsWith("SimpleName")) { + actionStr = actionStr.substring(index + 2); + if (actionStr.startsWith("MethodName")) { + singleEdit = singleEdit.replace("SimpleName", "MethodName"); + } else { + if (actionStr.startsWith("Name")) { + actionStr = actionStr.substring(5, 6); + if (!actionStr.equals(actionStr.toLowerCase())) { + singleEdit = singleEdit.replace("SimpleName", "Name"); + } else { + singleEdit = singleEdit.replace("SimpleName", "Variable"); + } + } else { + singleEdit = singleEdit.replace("SimpleName", "Variable"); + } + } + } + + editScript += singleEdit + " "; + } + actionSets.clear(); + actionSets.addAll(subSets); + } + return editScript; + } + + public String getAstEditScripts() { + return astEditScripts; + } + + public String getPatchesSourceCode() { + return patchesSourceCode; + } + + public String getBuggyTrees() { + return buggyTrees; + } + + public String getSizes() { + return sizes; + } + + public String getTokensOfSourceCode() { + return tokensOfSourceCode; + } + + public String getOriginalTree() { + return originalTree; + } + + public String getActionSets() { + return actionSets; + } +} diff --git a/src/main/java/edu/lu/uni/serval/violation/Alarm.java b/src/main/java/edu/lu/uni/serval/violation/Alarm.java new file mode 100644 index 0000000..2fe759c --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/violation/Alarm.java @@ -0,0 +1,66 @@ +package edu.lu.uni.serval.violation; + +import java.util.HashMap; +import java.util.Map; + +public class Alarm implements Comparable { + + private String buggyCommitId; + private String buggyFileName; + private Map positions; // : + private String fixedCommitId; + private String fixedFileName; + + public Alarm(String buggyCommitId, String buggyFileName, String fixedCommitId, + String fixedFileName) { + super(); + this.buggyCommitId = buggyCommitId; + this.buggyFileName = buggyFileName; + this.fixedCommitId = fixedCommitId; + this.fixedFileName = fixedFileName; + this.positions = new HashMap<>(); + } + + public String getBuggyCommitId() { + return buggyCommitId; + } + + public String getBuggyFileName() { + return buggyFileName; + } + + public Map getPositions() { + return positions; + } + + public String getFixedCommitId() { + return fixedCommitId; + } + + public String getFixedFileName() { + return fixedFileName; + } + + @Override + public int compareTo(Alarm a) { + int compareResult = this.buggyCommitId.compareTo(a.buggyCommitId); + if (compareResult == 0) { + compareResult = this.buggyFileName.compareTo(a.buggyFileName); + } + return compareResult; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof Alarm) { + Alarm alarm = (Alarm) obj; + if (alarm.buggyCommitId.equals(this.buggyCommitId) && alarm.buggyFileName.equals(this.buggyFileName) + && alarm.fixedCommitId.equals(this.fixedCommitId) && alarm.fixedFileName.equals(this.fixedFileName)) { + return true; + } + } + return false; + } + + +} diff --git a/src/main/java/edu/lu/uni/serval/violation/Violation.java b/src/main/java/edu/lu/uni/serval/violation/Violation.java new file mode 100644 index 0000000..0701034 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/violation/Violation.java @@ -0,0 +1,25 @@ +package edu.lu.uni.serval.violation; + +import java.util.ArrayList; +import java.util.List; + +public class Violation { + + private String project; + private List alarms; + + public Violation(String project) { + this.project = project; + this.alarms = new ArrayList<>(); + } + + public String getProject() { + return project; + } + + public List getAlarms() { + return alarms; + } + + +} diff --git a/src/main/java/edu/lu/uni/serval/violation/parse/AlarmsReader.java b/src/main/java/edu/lu/uni/serval/violation/parse/AlarmsReader.java new file mode 100644 index 0000000..3ea81fb --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/violation/parse/AlarmsReader.java @@ -0,0 +1,75 @@ +package edu.lu.uni.serval.violation.parse; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; + +import edu.lu.uni.serval.violation.Alarm; +import edu.lu.uni.serval.violation.Violation; + +public class AlarmsReader { + + public Map readAlarmsList(String fileName) { + Map violations = new HashMap<>(); + FileInputStream fis = null; + Scanner scanner = null; + try { + fis = new FileInputStream(fileName); + scanner = new Scanner(fis); + + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + int arrowIndex = line.indexOf("=>"); + String buggyInfo = line.substring(0, arrowIndex); + String fixedInfo = line.substring(arrowIndex + 2); + String[] buggyElements = buggyInfo.split(":"); + String[] fixedElements = fixedInfo.split(":"); + + String projectName = buggyElements[0]; + if (!projectName.equals(fixedElements[0])) continue; + + String commitId = buggyElements[1]; + String buggyFile = buggyElements[2]; + int startLine = Integer.parseInt(buggyElements[3]); + int endLine = Integer.parseInt(buggyElements[4]); + String fixCommitId = fixedElements[1]; + String fixedFile = fixedElements[2]; + + Alarm alarm = new Alarm(commitId, buggyFile, fixCommitId, fixedFile); + + Violation violation; + if (violations.containsKey(projectName)) { + violation = violations.get(projectName); + } else { + violation = new Violation(projectName); + violations.put(projectName, violation); + } + List alarms = violation.getAlarms(); + int index = alarms.indexOf(alarm); + if (index != -1) { + Alarm tempA = alarms.get(index); + Map positions = tempA.getPositions(); + positions.put(startLine, endLine); + } else { + alarm.getPositions().put(startLine, endLine); + alarms.add(alarm); + } + + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + try { + scanner.close(); + fis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return violations; + } +} diff --git a/src/main/java/edu/lu/uni/serval/violation/parse/ViolationParser.java b/src/main/java/edu/lu/uni/serval/violation/parse/ViolationParser.java new file mode 100644 index 0000000..07a7f0f --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/violation/parse/ViolationParser.java @@ -0,0 +1,112 @@ +package edu.lu.uni.serval.violation.parse; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import edu.lu.uni.serval.git.exception.GitRepositoryNotFoundException; +import edu.lu.uni.serval.git.exception.NotValidGitRepositoryException; +import edu.lu.uni.serval.git.travel.GitRepository; +import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.violation.Alarm; +import edu.lu.uni.serval.violation.Violation; + +public class ViolationParser { + + private static final String REPO_PATH = "/Volumes/MacBook/repositories/"; + private static final String revisedFilesPath = "GumTreeInput/revFiles/"; + private static final String previousFilesPath = "GumTreeInput/prevFiles/"; + private static final String positions = "GumTreeInput/positions/"; + + public static void main(String[] args) { + List repositoriesList = new ArrayList<>(); + File repositories = new File(REPO_PATH); + File[] subFiles = repositories.listFiles(); + for (File subFile : subFiles) { // repos-a to u + if (subFile.isDirectory()) { + File[] repos = subFile.listFiles(); + for (File repo : repos) { + if (repo.isDirectory()) { + repositoriesList.add(repo); + } + } + } + } + String fixedAlarmFile = "Dataset/fixed-alarms.list.txt"; + + FileHelper.createDirectory(revisedFilesPath); + FileHelper.createDirectory(previousFilesPath); + FileHelper.createDirectory(positions); + + ViolationParser parser = new ViolationParser(); + parser.parseViolations(fixedAlarmFile, repositoriesList); + } + + public void parseViolations(String fixedAlarmFile, List repos) { + AlarmsReader reader = new AlarmsReader(); + Map violations = reader.readAlarmsList(fixedAlarmFile); + + for (Map.Entry entry : violations.entrySet()) { + String projectName = entry.getKey(); + String repoName = ""; + for (File repo : repos) { + if (repo.getName().equals(projectName)) { + repoName = repo.getPath() + "/"; + break; + } + } + if ("".equals(repoName)) continue; + Violation violation = entry.getValue(); + List alarms = violation.getAlarms(); + + String repoPath = repoName + "/.git"; + GitRepository gitRepo = new GitRepository(repoPath, revisedFilesPath, previousFilesPath); + try { + gitRepo.open(); + for (Alarm alarm : alarms) { + String buggyCommitId = alarm.getBuggyCommitId(); + String buggyFileName = alarm.getBuggyFileName(); + String buggyFileContent = gitRepo.getFileContentByCommitIdAndFileName(buggyCommitId, buggyFileName); + if (buggyFileContent == null || "".equals(buggyFileContent)) continue; + + String fixedCommitId = alarm.getFixedCommitId(); + String fixedFileName = alarm.getFixedFileName(); + String fixedFileContent = gitRepo.getFileContentByCommitIdAndFileName(fixedCommitId, fixedFileName); + if (fixedFileContent == null || "".equals(fixedFileContent)) continue; + + String commitId = buggyCommitId.substring(0, 6) + "_" + fixedCommitId.substring(0, 6); + String fileName = fixedFileName.replaceAll("/", "#"); + fileName = projectName + "_" + commitId + fileName; + if (fileName.length() > 240) { + List files = FileHelper.getAllFilesInCurrentDiectory(revisedFilesPath, ".java"); + fileName = files.size() + "TooLongFileName.java"; + } + String buggyFile = previousFilesPath + "prev_" + fileName; + String fixedFile = revisedFilesPath + fileName; + String positionFile = positions + fileName.replace(".java", ".txt"); + FileHelper.outputToFile(buggyFile, buggyFileContent, false); + FileHelper.outputToFile(fixedFile, fixedFileContent, false); + FileHelper.outputToFile(positionFile, readPosition(alarm.getPositions()), false); + } + } catch (GitRepositoryNotFoundException e) { + e.printStackTrace(); + } catch (NotValidGitRepositoryException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + gitRepo.close(); + } + } + } + + private String readPosition(Map positions) { + String positionsStr = ""; + for (Map.Entry entry : positions.entrySet()) { + positionsStr += entry.getKey() + ":" + entry.getValue() + "\n"; + } + return positionsStr; + } +}