commit 229f103f079f403e01b390ac48e3f22773b8f3af Author: Kui LIU Date: Fri Jul 21 22:10:00 2017 +0200 Initialization. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..87eb87b --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +.cache-main +.DS_Store +.classpath +.project +.settings +logs +target +OUTPUT/ +*.class +*.list +*.csv +*.pdf + +# Package Files # +*.jar diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..a5685ff --- /dev/null +++ b/pom.xml @@ -0,0 +1,71 @@ + + 4.0.0 + + edu.lu.uni.serval + FixPatternMiner + 0.0.1-SNAPSHOT + jar + + FixPatternMiner + http://maven.apache.org + + + UTF-8 + 2.4.11 + + + + + + com.typesafe.akka + akka-actor_2.11 + ${akka.version} + + + + com.github.gumtreediff + core + 2.0.0-SNAPSHOT + + + + com.github.gumtreediff + gen.jdt + 2.0.0-SNAPSHOT + + + + edu.lu.uni.serval + SourceCodeParser + 0.0.1-SNAPSHOT + + + + edu.lu.uni.serval + GitTraveller + 0.0.1-SNAPSHOT + + + + junit + junit + 4.12 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + 1.8 + 1.8 + + + + + diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMiner/AkkaMiner.java b/src/main/java/edu/lu/uni/serval/FixPatternMiner/AkkaMiner.java new file mode 100644 index 0000000..46e24e8 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternMiner/AkkaMiner.java @@ -0,0 +1,60 @@ +package edu.lu.uni.serval.FixPatternMiner; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import edu.lu.uni.serval.utils.FileHelper; + +public class AkkaMiner { + + @SuppressWarnings("deprecation") + public static void main(String[] args) { + // input data + final List msgFiles = getMessageFiles(); + + // output path + final String editScriptsFilePath = "../GumTreeResults/editScripts/"; + final String patchesSourceCodeFilePath = "../GumTreeResults/sourceCode/"; + FileHelper.deleteDirectory(editScriptsFilePath); + FileHelper.deleteDirectory(patchesSourceCodeFilePath); + + ActorSystem system = null; + ActorRef parsingActor = null; + final int numberOfWorkers = 100; + try { + system = ActorSystem.create("Mining-FixPattern-System"); + parsingActor = system.actorOf(MineFixPatternActor.props(numberOfWorkers, editScriptsFilePath, patchesSourceCodeFilePath), "mine-fix-pattern-actor"); + parsingActor.tell(msgFiles, ActorRef.noSender()); + } catch (Exception e) { + system.shutdown(); + e.printStackTrace(); + } + } + + private static List getMessageFiles() { + String inputPath = "../OUTPUT/"; //DiffEntries prevFiles revFiles + File inputFileDirector = new File(inputPath); + File[] files = inputFileDirector.listFiles(); // project folders + List msgFiles = new ArrayList<>(); + + for (File file : files) { + String projectFolder = file.getPath(); + File revFileFolder = new File(projectFolder + "/revFiles/");// revised file folder + File[] revFiles = revFileFolder.listFiles(); + + for (File revFile : revFiles) { + if (revFile.getName().endsWith(".java")) { + File prevFile = new File(projectFolder + "/prevFiles/prev_" + revFile.getName());// previous file + File diffentryFile = new File(projectFolder + "/DiffEntries/" + revFile.getName().replace(".java", ".txt")); // DiffEntry file + MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); + msgFiles.add(msgFile); + } + } + } + + return msgFiles; + } +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMiner/App.java b/src/main/java/edu/lu/uni/serval/FixPatternMiner/App.java new file mode 100644 index 0000000..e3d2323 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternMiner/App.java @@ -0,0 +1,13 @@ +package edu.lu.uni.serval.FixPatternMiner; + +/** + * Hello world! + * + */ +public class App +{ + public static void main( String[] args ) + { + System.out.println( "Hello World!" ); + } +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMiner/MessageFile.java b/src/main/java/edu/lu/uni/serval/FixPatternMiner/MessageFile.java new file mode 100644 index 0000000..1f5399d --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternMiner/MessageFile.java @@ -0,0 +1,30 @@ +package edu.lu.uni.serval.FixPatternMiner; + +import java.io.File; + +public class MessageFile { + + private File revFile; + private File prevFile; + private File diffEntryFile; + + public MessageFile(File revFile, File prevFile, File diffEntryFile) { + super(); + this.revFile = revFile; + this.prevFile = prevFile; + this.diffEntryFile = diffEntryFile; + } + + public File getRevFile() { + return revFile; + } + + public File getPrevFile() { + return prevFile; + } + + public File getDiffEntryFile() { + return diffEntryFile; + } + +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMiner/MineFixPatternActor.java b/src/main/java/edu/lu/uni/serval/FixPatternMiner/MineFixPatternActor.java new file mode 100644 index 0000000..83c25ff --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternMiner/MineFixPatternActor.java @@ -0,0 +1,76 @@ +package edu.lu.uni.serval.FixPatternMiner; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import akka.actor.ActorRef; +import akka.actor.Props; +import akka.actor.UntypedActor; +import akka.japi.Creator; +import akka.routing.RoundRobinPool; + +public class MineFixPatternActor extends UntypedActor { + + private static Logger logger = LoggerFactory.getLogger(MineFixPatternActor.class); + + private ActorRef mineRouter; + private final int numberOfWorkers; + private int counter = 0; + + public MineFixPatternActor(int numberOfWorkers, String editScriptsFilePath, String patchesSourceCodeFilePath) { + mineRouter = this.getContext().actorOf(new RoundRobinPool(numberOfWorkers) + .props(MineFixPatternWorker.props(editScriptsFilePath, patchesSourceCodeFilePath)), "mine-fix-pattern-router"); + this.numberOfWorkers = numberOfWorkers; + } + + public static Props props(final int numberOfWorkers, final String editScriptsFilePath, final String patchesSourceCodeFilePath) { + + return Props.create(new Creator() { + + private static final long serialVersionUID = 9207427376110704705L; + + @Override + public MineFixPatternActor create() throws Exception { + return new MineFixPatternActor(numberOfWorkers, editScriptsFilePath, patchesSourceCodeFilePath); + } + + }); + } + + @SuppressWarnings("deprecation") + @Override + public void onReceive(Object message) throws Exception { + if (message instanceof List) { + List files = (List) message; + int size = files.size(); + int average = size / numberOfWorkers; + + for (int i = 0; i < numberOfWorkers; i ++) { + int fromIndex = i * average; + int toIndex = (i + 1) * average; + if (i == numberOfWorkers - 1) { + toIndex = size; + } + + List filesOfWorkers = new ArrayList<>(); + filesOfWorkers.addAll(files.subList(fromIndex, toIndex)); + mineRouter.tell(filesOfWorkers, getSelf()); + logger.info("Assign a task to worker #" + (i + 1) + "..."); + } + } else if ("STOP".equals(message.toString())) { + counter ++; + logger.info("Worker #" + counter + " finished the work..."); + if (counter >= numberOfWorkers) { + this.getContext().stop(mineRouter); + this.getContext().stop(getSelf()); + this.getContext().system().shutdown(); + } + } else { + unhandled(message); + } + } + +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMiner/MineFixPatternWorker.java b/src/main/java/edu/lu/uni/serval/FixPatternMiner/MineFixPatternWorker.java new file mode 100644 index 0000000..3a3ee81 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternMiner/MineFixPatternWorker.java @@ -0,0 +1,58 @@ +package edu.lu.uni.serval.FixPatternMiner; + +import java.io.File; +import java.util.List; + +import akka.actor.Props; +import akka.actor.UntypedActor; +import akka.japi.Creator; +import edu.lu.uni.serval.utils.FileHelper; + +public class MineFixPatternWorker extends UntypedActor { + + private String editScriptsFilePath; + private String patchesSourceCodeFilePath; + + public MineFixPatternWorker(String editScriptsFilePath, String patchesSourceCodeFilePath) { + this.editScriptsFilePath = editScriptsFilePath; + this.patchesSourceCodeFilePath = patchesSourceCodeFilePath; + } + + public static Props props(final String editScriptsFile, final String patchesSourceCodeFile) { + return Props.create(new Creator() { + + private static final long serialVersionUID = -7615153844097275009L; + + @Override + public MineFixPatternWorker create() throws Exception { + return new MineFixPatternWorker(editScriptsFile, patchesSourceCodeFile); + } + + }); + } + + @Override + public void onReceive(Object message) throws Exception { + if (message instanceof List) { + List files = (List) message; + StringBuilder editScripts = new StringBuilder(); + StringBuilder patchesSourceCode = new StringBuilder(); + for (Object obj : files) { + MessageFile msgFile = (MessageFile) obj; + Miner miner = new Miner(); + miner.mineFixPatterns(msgFile.getPrevFile(), msgFile.getRevFile(), msgFile.getDiffEntryFile()); + editScripts.append(miner.getAstEditScripts()); + patchesSourceCode.append(miner.getPatchesSourceCode()); + } + + List subFiles1 = FileHelper.getAllFilesInCurrentDiectory(editScriptsFilePath, ".list"); + List subFiles2 = FileHelper.getAllFilesInCurrentDiectory(patchesSourceCodeFilePath, ".list"); + FileHelper.outputToFile(editScriptsFilePath + "edistScripts" + subFiles1.size() + ".list", editScripts, false); + FileHelper.outputToFile(patchesSourceCodeFilePath + "patches" + subFiles2.size() + ".list", patchesSourceCode, false); + + this.getSender().tell("STOP", getSelf()); + } else { + unhandled(message); + } + } +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMiner/Miner.java b/src/main/java/edu/lu/uni/serval/FixPatternMiner/Miner.java new file mode 100644 index 0000000..6a2fefb --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternMiner/Miner.java @@ -0,0 +1,343 @@ +package edu.lu.uni.serval.FixPatternMiner; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.jdt.core.dom.CompilationUnit; + +import com.github.gumtreediff.actions.model.Move; +import com.github.gumtreediff.actions.model.Update; +import com.github.gumtreediff.tree.ITree; + +import edu.lu.uni.serval.gumtree.GumTreeComparer; +import edu.lu.uni.serval.gumtree.regroup.ActionFilter; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.gumtree.regroup.SimpleTree; +import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; +import edu.lu.uni.serval.utils.CUCreator; +import edu.lu.uni.serval.utils.FileHelper; + +/** + * Mine fix patterns from GumTree results. + * + * @author kui.liu + * + */ +public class Miner { + private String astEditScripts = ""; + private String patchesSourceCode = ""; + + public void mineFixPatterns(File prevFile, File revFile, File diffEntryFile) throws FileNotFoundException, IOException { + // GumTree results + List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); + + // Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part. + List hierarchicalActionSets = new ActionFilter().filterOutUselessActions(gumTreeResults); + + if (hierarchicalActionSets.size() > 0) { + CUCreator cuCreator = new CUCreator(); + CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); + CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); + + String sourceCode = FileHelper.readFile(diffEntryFile); + for (HierarchicalActionSet actionSet : hierarchicalActionSets) { + // position of buggy statements + int startPosition = 0; + int endPosition = 0; + // position of fixed statements + int startPosition2 = 0; + int endPosition2 = 0; + + String actionStr = actionSet.getActionString(); + String astNodeType = actionSet.getAstNodeType(); + if (actionStr.startsWith("INS")) { + startPosition2 = actionSet.getStartPosition(); + endPosition2 = startPosition2 + actionSet.getLength(); + + if ("EnhancedForStatement".equals(astNodeType) || "ForStatement".equals(astNodeType) + || "DoStatement".equals(astNodeType) || "WhileStatement".equals(astNodeType) + || "LabeledStatement".equals(astNodeType) || "SynchronizedStatement".equals(astNodeType) + || "IfStatement".equals(astNodeType) || "TryStatement".equals(astNodeType)) { + List firstAndLastMov = getFirstAndLastMoveAction(actionSet); + if (firstAndLastMov != null) { + startPosition = firstAndLastMov.get(0).getNode().getPos(); + ITree lastTree = firstAndLastMov.get(1).getNode(); + endPosition = lastTree.getPos() + lastTree.getLength(); + } else { // Pure insert actions without any move actions. + continue; + } + } else { // other insert statements + continue; + } + } else if (actionStr.startsWith("UPD")) { + startPosition = actionSet.getStartPosition(); + endPosition = startPosition + actionSet.getLength(); + Update update = (Update) actionSet.getAction(); + ITree newNode = update.getNewNode(); + startPosition2 = newNode.getPos(); + endPosition2 = startPosition2 + newNode.getLength(); + + if ("EnhancedForStatement".equals(astNodeType) || "ForStatement".equals(astNodeType) + || "DoStatement".equals(astNodeType) || "WhileStatement".equals(astNodeType) + || "LabeledStatement".equals(astNodeType) || "SynchronizedStatement".equals(astNodeType) + || "IfStatement".equals(astNodeType) || "TryStatement".equals(astNodeType)) { + List children = update.getNode().getChildren(); + endPosition = getEndPosition(children); + List newChildren = newNode.getChildren(); + endPosition2 = getEndPosition(newChildren); + } + } else {// DEL actions and MOV actions: we don't need these actions, as for now. + continue; + } + + // Get the buggy code and fixed code + if (startPosition != 0 && startPosition2 != 0) { + // Line numbers of buggy statements + int startLineNum = prevUnit.getLineNumber(startPosition); + int endLineNum = prevUnit.getLineNumber(endPosition); + // Line numbers of fixed statements + int startLineNum2 = revUnit.getLineNumber(startPosition2); + int endLineNum2 = revUnit.getLineNumber(endPosition2); + + // Limit the range of buggy code and fixed code. TODO: + + actionSet.setStartLineNum(startLineNum); + actionSet.setEndLineNum(endLineNum); + + // Source Code of patches. + String patchSourceCode = getPatchSourceCode(sourceCode, startLineNum, endLineNum, startLineNum2, + endLineNum2); + if (patchSourceCode != null) { + patchesSourceCode += "PATCH###Num\n" + patchSourceCode + "\n"; + + /** + * Convert the ITree of buggy code to a simple tree. + * TODO: it will be used to compute the similarity. + */ + SimplifyTree abstractIdentifier = new SimplifyTree(); + abstractIdentifier.abstractTree(actionSet); + SimpleTree simpleTree = actionSet.getSimpleTree(); + SimpleTree abstractSimpleTree = actionSet.getAbstractSimpleTree(); + clearITree(actionSet); + + /** + * Select edit scripts for deep learning. + * Edit scripts will be used to mine common fix patterns. + */ + // 1. First level: AST node type. + String astEditScripts = getASTEditScripts(actionSet); + this.astEditScripts += astEditScripts + "\n"; + // 2. source code: raw tokens + String rawTokenEditScripts = getRawTokenEditScripts(actionSet); + // 3. abstract identifiers: + String abstractIdentifiersEditScripts = getAbstractIdentifiersEditScripts(actionSet); + // 4. semi-source code: + String semiSourceCodeEditScripts = getSemiSourceCodeEditScripts(actionSet); + } + } + } + } + } + + private String getSemiSourceCodeEditScripts(HierarchicalActionSet actionSet) { + // TODO Auto-generated method stub + return null; + } + + private String getAbstractIdentifiersEditScripts(HierarchicalActionSet actionSet) { + // TODO Auto-generated method stub + return null; + } + + private String getRawTokenEditScripts(HierarchicalActionSet actionSet) { + // TODO Auto-generated method stub + return null; + } + + private int getEndPosition(List children) { + int endPosition = 0; + for (ITree child : children) { + if (child.getLabel().endsWith("Body")) { + endPosition = child.getPos() - 1; + break; + } + } + return endPosition; + } + + private List getFirstAndLastMoveAction(HierarchicalActionSet gumTreeResult) { + List firstAndLastMoveActions = new ArrayList<>(); + List actions = gumTreeResult.getSubActions(); + if (actions.size() == 0) { + return null; + } + Move firstMoveAction = null; + Move lastMoveAction = null; + while (actions.size() > 0) { + List subActions = new ArrayList<>(); + for (HierarchicalActionSet action : actions) { + subActions.addAll(action.getSubActions()); + if (action.toString().startsWith("MOV")) { + if (firstMoveAction == null) { + firstMoveAction = (Move) action.getAction(); + lastMoveAction = (Move) action.getAction(); + } else { + int startPosition = action.getStartPosition(); + int length = action.getLength(); + int startPositionFirst = firstMoveAction.getPosition(); + int startPositionLast = lastMoveAction.getPosition(); + int lengthLast = lastMoveAction.getNode().getLength(); + if (startPosition < startPositionFirst) { + firstMoveAction = (Move) action.getAction(); + } else if ((startPosition + length) > (startPositionLast + lengthLast)) { + lastMoveAction = (Move) action.getAction(); + } + } + } + } + + actions.clear(); + actions.addAll(subActions); + } + if (firstMoveAction == null) { + return null; + } + firstAndLastMoveActions.add(firstMoveAction); + firstAndLastMoveActions.add(lastMoveAction); + return firstAndLastMoveActions; + } + + private String getPatchSourceCode(String sourceCode, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) { + String buggyStatements = ""; + String fixedStatements = ""; + BufferedReader reader = null; + try { + reader = new BufferedReader(new StringReader(sourceCode)); + String line = null; + int startLine = 0; + int counter = 0; + int range = 0; + int startLine2 = 0; + int counter2 = 0; + int range2 = 0; + int counter3 = 0; // counter of non-buggy code line. + while ((line = reader.readLine()) != null) { + if (startLine == 0 && line.startsWith("@@ -")) { + String lineNum = line.substring(4); + lineNum = lineNum.substring(0, lineNum.indexOf(" ")); + String[] nums = lineNum.split(","); + startLine = Integer.parseInt(nums[0].trim()); + range = Integer.parseInt(nums[1].trim()); + if (startLine > startLineNum) { + return null; // Wrong Matching. + } + if (startLine + range < startLineNum) { + startLine = 0; + continue; + } + String lineNum2 = line.substring(line.indexOf("+")).trim(); + lineNum2 = lineNum2.substring(1, lineNum2.length() - 2); + String[] nums2 = lineNum2.split(","); + startLine2 = Integer.parseInt(nums2[0].trim()); + range2 = Integer.parseInt(nums2[1].trim()); + continue; + } + + int lineNum1 = counter + counter3; + int lineNum2 = counter2 + counter3; + if (startLine > 0 && lineNum1 < range && lineNum2 < range2) { + if (line.startsWith("-") && startLine + lineNum1 >= startLineNum && startLine + lineNum1 <= endLineNum) { + buggyStatements += line + "\n"; + } else if (line.startsWith("+") && startLine2 + lineNum2 >= startLineNum2 && startLine2 + lineNum2 <= endLineNum2) { + fixedStatements += line + "\n"; + } + if (line.startsWith("-")) { + counter ++; + } else if (line.startsWith("+")) { + counter2 ++; + } else { + counter3 ++; + } + } + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + if (reader != null) { + reader.close(); + reader = null; + } + } catch (IOException e) { + e.printStackTrace(); + } + } + return buggyStatements + "\n" + fixedStatements; + } + + /** + * Get the AST node based edit script of patches in terms of breadth first. + * + * @param actionSet + * @return + */ + private String getASTEditScripts(HierarchicalActionSet actionSet) { + String editScript = ""; + + List actionSets = new ArrayList<>(); + actionSets.add(actionSet); + while (actionSets.size() != 0) { + List subSets = new ArrayList<>(); + for (HierarchicalActionSet set : actionSets) { + subSets.addAll(set.getSubActions()); + String actionStr = set.getActionString(); + int index = actionStr.indexOf("@@"); + String singleEdit = actionStr.substring(0, index).replace(" ", ""); + + if (singleEdit.endsWith("SimpleName")) { + actionStr = actionStr.substring(index + 2); + if (actionStr.startsWith("MethodName")) { + singleEdit = singleEdit.replace("SimpleName", "MethodName"); + } else { + if (actionStr.startsWith("Name")) { + actionStr = actionStr.substring(5, 6); + if (!actionStr.equals(actionStr.toLowerCase())) { + singleEdit = singleEdit.replace("SimpleName", "Name"); + } else { + singleEdit = singleEdit.replace("SimpleName", "Variable"); + } + } else { + singleEdit = singleEdit.replace("SimpleName", "Variable"); + } + } + } + + editScript += singleEdit + " "; + } + actionSets.clear(); + actionSets.addAll(subSets); + } + return editScript; + } + + private void clearITree(HierarchicalActionSet actionSet) { + actionSet.getAction().setNode(null); + for (HierarchicalActionSet subActionSet : actionSet.getSubActions()) { + clearITree(subActionSet); + } + } + + public String getAstEditScripts() { + return astEditScripts; + } + + public String getPatchesSourceCode() { + return patchesSourceCode; + } + +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternMiner/Miner2.java b/src/main/java/edu/lu/uni/serval/FixPatternMiner/Miner2.java new file mode 100644 index 0000000..29d6a48 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternMiner/Miner2.java @@ -0,0 +1,338 @@ +package edu.lu.uni.serval.FixPatternMiner; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.jdt.core.dom.CompilationUnit; + +import com.github.gumtreediff.actions.model.Move; +import com.github.gumtreediff.actions.model.Update; +import com.github.gumtreediff.tree.ITree; + +import edu.lu.uni.serval.gumtree.GumTreeComparer; +import edu.lu.uni.serval.gumtree.regroup.ActionFilter; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.gumtree.regroup.SimpleTree; +import edu.lu.uni.serval.gumtree.regroup.SimplifyTree; +import edu.lu.uni.serval.utils.CUCreator; +import edu.lu.uni.serval.utils.FileHelper; + +public class Miner2 { + + public static void main(String[] args) { + try { + inputData(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public static void inputData() throws FileNotFoundException, IOException { + String inputPath = "../../OUTPUT/"; //DiffEntries prevFiles revFiles + File inputFileDirector = new File(inputPath); + File[] files = inputFileDirector.listFiles(); // project folders + + FileHelper.deleteDirectory("../../GumTreeResults/Exp/"); + FileHelper.deleteDirectory("../../GumTreeResults/Exp_ASTNode/"); + FileHelper.deleteDirectory("../../GumTreeResults/Exp_RawCode/"); + + StringBuilder astEditScriptsBuilder = new StringBuilder(); + + for (File file : files) { + String projectFolder = file.getPath(); + File revFileFolder = new File(projectFolder + "/revFiles/");// revised file folder + File[] revFiles = revFileFolder.listFiles(); + + for (File revFile : revFiles) { + if (revFile.getName().endsWith(".java")) { + File prevFile = new File(projectFolder + "/prevFiles/prev_" + revFile.getName());// previous file + File diffentryFile = new File(projectFolder + "/DiffEntries/" + revFile.getName().replace(".java", ".txt")); // DiffEntry file + + // GumTree results + List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); + // Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part. + gumTreeResults = new ActionFilter().filterOutUselessActions(gumTreeResults); + if (gumTreeResults.size() > 0) { + CUCreator cuCreator = new CUCreator(); + CompilationUnit prevUnit = cuCreator.createCompilationUnit(prevFile); + CompilationUnit revUnit = cuCreator.createCompilationUnit(revFile); + + // Commit Message TODO + // COMMIT_MSG#Num. + String sourceCode = FileHelper.readFile(diffentryFile); + for (HierarchicalActionSet gumTreeResult : gumTreeResults) { + // set line numbers + // position of buggy statements + int startPosition = 0; + int endPosition = 0; + // position of fixed statements + int startPosition2 = 0; + int endPosition2 = 0; + + String actionStr = gumTreeResult.getActionString(); + if (actionStr.startsWith("INS")) { + startPosition2 = gumTreeResult.getStartPosition(); + endPosition2 = startPosition2 + gumTreeResult.getLength(); + + if (actionStr.startsWith("INS LabeledStatement") || actionStr.startsWith("INS ForStatement") + || actionStr.startsWith("INS EnhancedForStatement") || actionStr.startsWith("INS DoStatement") + || actionStr.startsWith("INS WhileStatement") || actionStr.startsWith("INS SynchronizedStatement") + || actionStr.startsWith("INS IfStatement") || actionStr.startsWith("INS TryStatement")) { + List firstAndLastMov = getFirstAndLastMoveAction(gumTreeResult); + if (firstAndLastMov != null) { + startPosition = firstAndLastMov.get(0).getNode().getPos(); + endPosition = firstAndLastMov.get(1).getNode().getPos() + firstAndLastMov.get(1).getNode().getLength(); + } else { // Pure insert actions without any move actions. + continue; + } + } else { // other insert statements + continue; + } + } else if (actionStr.startsWith("UPD")) { + startPosition = gumTreeResult.getStartPosition(); + endPosition = startPosition + gumTreeResult.getLength(); + Update update = (Update) gumTreeResult.getAction(); + ITree newNode = update.getNewNode(); + startPosition2 = newNode.getPos(); + endPosition2 = startPosition2 + newNode.getLength(); + + if (actionStr.startsWith("UPD LabeledStatement") || actionStr.startsWith("UPD ForStatement") + || actionStr.startsWith("UPD EnhancedForStatement") || actionStr.startsWith("UPD DoStatement") + || actionStr.startsWith("UPD WhileStatement") || actionStr.startsWith("UPD SynchronizedStatement") + || actionStr.startsWith("UPD IfStatement") || actionStr.startsWith("UPD TryStatement")) { + List children = gumTreeResult.getAction().getNode().getChildren(); + endPosition = getEndPosition(children); + List newChildren = newNode.getChildren(); + endPosition2 = getEndPosition(newChildren); + } + } else {// DEL actions and MOV actions: we don't need these actions, as for now. + continue; + } + + // Get the buggy code and fixed code + if (startPosition != 0 && startPosition2 != 0) { + // Line numbers of buggy statements + int startLineNum = prevUnit.getLineNumber(startPosition); + int endLineNum = prevUnit.getLineNumber(endPosition); + // Line numbers of fixed statements + int startLineNum2 = revUnit.getLineNumber(startPosition2); + int endLineNum2 = revUnit.getLineNumber(endPosition2); + + // Limit the range of buggy code and fixed code. TODO: + + gumTreeResult.setStartLineNum(startLineNum); + gumTreeResult.setEndLineNum(endLineNum); + + // Source Code of patches. + String patchSourceCode = getPatchSourceCode(sourceCode, startLineNum, endLineNum, startLineNum2, endLineNum2); + if (patchSourceCode != null) { + patchSourceCode = "PATCH###Num\n" + patchSourceCode + "\n"; + + /** + * Simple tree of buggy code. + * TODO: it will be used to compute the similarity. + * Or re-parse the buggy code to get the simple tree. + */ + // convert the ITree of buggy code to a simple tree. + SimplifyTree abstractIdentifier = new SimplifyTree(); + abstractIdentifier.abstractTree(gumTreeResult); + SimpleTree simpleTree = gumTreeResult.getSimpleTree(); + SimpleTree abstractSimpleTree = gumTreeResult.getAbstractSimpleTree(); + clearITree(gumTreeResult); + + /** + * Select edit scripts for deep learning. + * Edit scripts will be used to mine common fix patterns. + */ + // 1. First level: AST node type. + String astEditScripts = getASTEditScripts(gumTreeResult); + astEditScriptsBuilder.append(astEditScripts + "\n"); + + // 2. source code TODO + // 3. abstract identifiers TODO + // 4. semi-source code. TODO + } + } + } + } + } + } + + FileHelper.outputToFile("../../GumTreeResults/Exp_ASTNode/EditScripts.list", astEditScriptsBuilder, true); + astEditScriptsBuilder.setLength(0); + + System.out.println(file); + } + + } + + private static int getEndPosition(List children) { + int endPosition = 0; + for (ITree child : children) { + if (child.getLabel().endsWith("Body")) { + endPosition = child.getPos() - 1; + break; + } + } + return endPosition; + } + + private static List getFirstAndLastMoveAction(HierarchicalActionSet gumTreeResult) { + List firstAndLastMoveActions = new ArrayList<>(); + List actions = gumTreeResult.getSubActions(); + if (actions.size() == 0) { + return null; + } + Move firstMoveAction = null; + Move lastMoveAction = null; + while (actions.size() > 0) { + List subActions = new ArrayList<>(); + for (HierarchicalActionSet action : actions) { + subActions.addAll(action.getSubActions()); + if (action.toString().startsWith("MOV")) { + if (firstMoveAction == null) { + firstMoveAction = (Move) action.getAction(); + } + lastMoveAction = (Move) action.getAction(); + } + } + + actions.clear(); + actions.addAll(subActions); + } + if (firstMoveAction == null) { + return null; + } + firstAndLastMoveActions.add(firstMoveAction); + firstAndLastMoveActions.add(lastMoveAction); + return firstAndLastMoveActions; + } + + private static String getPatchSourceCode(String sourceCode, int startLineNum, int endLineNum, int startLineNum2, int endLineNum2) { + String buggyStatements = ""; + String fixedStatements = ""; + BufferedReader reader = null; + try { + reader = new BufferedReader(new StringReader(sourceCode)); + String line = null; + int startLine = 0; + int counter = 0; + int range = 0; + int startLine2 = 0; + int counter2 = 0; + int range2 = 0; + int counter3 = 0; // counter of non-buggy code line. + while ((line = reader.readLine()) != null) { + if (startLine == 0 && line.startsWith("@@ -")) { + String lineNum = line.substring(4); + lineNum = lineNum.substring(0, lineNum.indexOf(" ")); + String[] nums = lineNum.split(","); + startLine = Integer.parseInt(nums[0].trim()); + range = Integer.parseInt(nums[1].trim()); + if (startLine > startLineNum) { + return null; // Wrong Matching. + } + if (startLine + range < startLineNum) { + startLine = 0; + continue; + } + String lineNum2 = line.substring(line.indexOf("+")).trim(); + lineNum2 = lineNum2.substring(1, lineNum2.length() - 2); + String[] nums2 = lineNum2.split(","); + startLine2 = Integer.parseInt(nums2[0].trim()); + range2 = Integer.parseInt(nums2[1].trim()); + continue; + } + + int lineNum1 = counter + counter3; + int lineNum2 = counter2 + counter3; + if (startLine > 0 && lineNum1 < range && lineNum2 < range2) { + if (line.startsWith("-") && startLine + lineNum1 >= startLineNum && startLine + lineNum1 <= endLineNum) { + buggyStatements += line + "\n"; + } else if (line.startsWith("+") && startLine2 + lineNum2 >= startLineNum2 && startLine2 + lineNum2 <= endLineNum2) { + fixedStatements += line + "\n"; + } + if (line.startsWith("-")) { + counter ++; + } else if (line.startsWith("+")) { + counter2 ++; + } else { + counter3 ++; + } + } + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + if (reader != null) { + reader.close(); + reader = null; + } + } catch (IOException e) { + e.printStackTrace(); + } + } + return buggyStatements + "\n" + fixedStatements; + } + + /** + * Get the AST node based edit script of patches in terms of breadth first. + * + * @param actionSet + * @return + */ + private static String getASTEditScripts(HierarchicalActionSet actionSet) { + String editScript = ""; + + List actionSets = new ArrayList<>(); + actionSets.add(actionSet); + while (actionSets.size() != 0) { + List subSets = new ArrayList<>(); + for (HierarchicalActionSet set : actionSets) { + subSets.addAll(set.getSubActions()); + String actionStr = set.getActionString(); + int index = actionStr.indexOf("@@"); + String singleEdit = actionStr.substring(0, index).replace(" ", ""); + + if (singleEdit.endsWith("SimpleName")) { + actionStr = actionStr.substring(index + 2); + if (actionStr.startsWith("MethodName")) { + singleEdit = singleEdit.replace("SimpleName", "MethodName"); + } else { + if (actionStr.startsWith("Name")) { + actionStr = actionStr.substring(5, 6); + if (!actionStr.equals(actionStr.toLowerCase())) { + singleEdit = singleEdit.replace("SimpleName", "Name"); + } else { + singleEdit = singleEdit.replace("SimpleName", "Variable"); + } + } else { + singleEdit = singleEdit.replace("SimpleName", "Variable"); + } + } + } + + editScript += singleEdit + " "; + } + actionSets.clear(); + actionSets.addAll(subSets); + } + return editScript; + } + + private static void clearITree(HierarchicalActionSet actionSet) { + actionSet.getAction().setNode(null); + for (HierarchicalActionSet subActionSet : actionSet.getSubActions()) { + clearITree(subActionSet); + } + } + +} diff --git a/src/test/java/edu/lu/uni/serval/FixPatternMiner/AppTest.java b/src/test/java/edu/lu/uni/serval/FixPatternMiner/AppTest.java new file mode 100644 index 0000000..ceeac95 --- /dev/null +++ b/src/test/java/edu/lu/uni/serval/FixPatternMiner/AppTest.java @@ -0,0 +1,38 @@ +package edu.lu.uni.serval.FixPatternMiner; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class AppTest + extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public AppTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( AppTest.class ); + } + + /** + * Rigourous Test :-) + */ + public void testApp() + { + assertTrue( true ); + } +}