From 3d20268377926343ba99434cf8a7b4843409118f Mon Sep 17 00:00:00 2001 From: fixminer Date: Mon, 17 Sep 2018 21:20:13 +0200 Subject: [PATCH 1/4] rc 3 --- .../serval/FixPatternParser/CUCreator.java | 55 ------------------- ...arser.java => FixedPatternHunkParser.java} | 6 +- ...ionParser.java => FixedPatternParser.java} | 7 +-- .../HierarchicalActionSet.java | 2 +- .../HierarchicalRegrouper.java | 4 +- .../edu/lu/uni/serval/fixminer/Launcher.java | 3 +- .../{cluster => }/MultiThreadTreeLoader.java | 6 +- .../fixminer/TestHunkParserSingleFile.java | 12 ++-- .../uni/serval/fixminer/cluster/Compare.java | 2 +- .../serval/fixminer/cluster/WorkMessage.java | 51 ----------------- .../fixminer/cluster/akka/AkkaTreeParser.java | 1 - .../cluster/akka/EDiffActor.java} | 22 ++++---- .../cluster/akka/EDiffMessage.java} | 6 +- .../cluster/akka/EDiffWorker.java} | 22 ++++---- .../cluster/akka}/MessageFile.java | 2 +- .../cluster/{ => akka}/TreeActor.java | 3 +- .../cluster/{ => akka}/TreeWorker.java | 7 ++- .../{cluster => jobs}/AkkaTreeLoader.java | 5 +- .../{cluster => jobs}/CalculatePairs.java | 5 +- .../fixminer/{ => jobs}/EnhancedASTDiff.java | 12 ++-- .../{cluster => jobs}/ImportPairs2DB.java | 3 +- .../MultiThreadTreeLoaderCluster.java | 11 ++-- .../MultiThreadTreeLoaderCluster3.java | 9 +-- .../fixminer/{ => jobs}/PatternExtractor.java | 6 +- .../{cluster => jobs}/StoreEDiffInCache.java | 9 +-- .../TreeLoaderClusterL1.java | 5 +- .../{FixPattern => }/utils/ASTNodeMap.java | 2 +- .../cluster => utils}/CallShell.java | 2 +- .../{FixPattern => }/utils/Checker.java | 2 +- .../serval/{FixPattern => }/utils/EDiff.java | 4 +- .../{FixPattern => }/utils/EDiffHelper.java | 4 +- .../{FixPattern => }/utils/PoolBuilder.java | 2 +- 32 files changed, 94 insertions(+), 198 deletions(-) delete mode 100644 src/main/java/edu/lu/uni/serval/FixPatternParser/CUCreator.java rename src/main/java/edu/lu/uni/serval/FixPatternParser/{violations/FixedViolationHunkParser.java => FixedPatternHunkParser.java} (92%) rename src/main/java/edu/lu/uni/serval/FixPatternParser/{violations/FixedViolationParser.java => FixedPatternParser.java} (86%) rename src/main/java/edu/lu/uni/serval/{gumtree/regroup => FixPatternParser}/HierarchicalActionSet.java (99%) rename src/main/java/edu/lu/uni/serval/{gumtree/regroup => FixPatternParser}/HierarchicalRegrouper.java (98%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster => }/MultiThreadTreeLoader.java (99%) delete mode 100644 src/main/java/edu/lu/uni/serval/fixminer/cluster/WorkMessage.java rename src/main/java/edu/lu/uni/serval/{MultipleThreadsParser/ParseFixPatternActor.java => fixminer/cluster/akka/EDiffActor.java} (70%) rename src/main/java/edu/lu/uni/serval/{MultipleThreadsParser/WorkMessage.java => fixminer/cluster/akka/EDiffMessage.java} (64%) rename src/main/java/edu/lu/uni/serval/{MultipleThreadsParser/ParseFixPatternWorker.java => fixminer/cluster/akka/EDiffWorker.java} (85%) rename src/main/java/edu/lu/uni/serval/{MultipleThreadsParser => fixminer/cluster/akka}/MessageFile.java (93%) rename src/main/java/edu/lu/uni/serval/fixminer/cluster/{ => akka}/TreeActor.java (95%) rename src/main/java/edu/lu/uni/serval/fixminer/cluster/{ => akka}/TreeWorker.java (96%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster => jobs}/AkkaTreeLoader.java (95%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster => jobs}/CalculatePairs.java (97%) rename src/main/java/edu/lu/uni/serval/fixminer/{ => jobs}/EnhancedASTDiff.java (87%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster => jobs}/ImportPairs2DB.java (95%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster => jobs}/MultiThreadTreeLoaderCluster.java (97%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster => jobs}/MultiThreadTreeLoaderCluster3.java (99%) rename src/main/java/edu/lu/uni/serval/fixminer/{ => jobs}/PatternExtractor.java (99%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster => jobs}/StoreEDiffInCache.java (95%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster => jobs}/TreeLoaderClusterL1.java (96%) rename src/main/java/edu/lu/uni/serval/{FixPattern => }/utils/ASTNodeMap.java (98%) rename src/main/java/edu/lu/uni/serval/{fixminer/cluster => utils}/CallShell.java (98%) rename src/main/java/edu/lu/uni/serval/{FixPattern => }/utils/Checker.java (98%) rename src/main/java/edu/lu/uni/serval/{FixPattern => }/utils/EDiff.java (96%) rename src/main/java/edu/lu/uni/serval/{FixPattern => }/utils/EDiffHelper.java (96%) rename src/main/java/edu/lu/uni/serval/{FixPattern => }/utils/PoolBuilder.java (95%) diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/CUCreator.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/CUCreator.java deleted file mode 100644 index ec47e1a..0000000 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/CUCreator.java +++ /dev/null @@ -1,55 +0,0 @@ -package edu.lu.uni.serval.FixPatternParser; - -import org.eclipse.jdt.core.dom.AST; -import org.eclipse.jdt.core.dom.ASTParser; -import org.eclipse.jdt.core.dom.CompilationUnit; - -import java.io.*; - -/** - * Creator of a CompilationUnit. - * - * @author kui.liu - * - */ -public class CUCreator { - - public CompilationUnit createCompilationUnit(File javaFile) { - CompilationUnit unit = null; - try { - char[] sourceCode = readFileToCharArray(new FileReader(javaFile)); - ASTParser parser = createASTParser(sourceCode); - parser.setKind(ASTParser.K_COMPILATION_UNIT); - unit = (CompilationUnit) parser.createAST(null); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - - return unit; - } - - private ASTParser createASTParser(char[] javaCode) { - ASTParser parser = ASTParser.newParser(AST.JLS8); - parser.setSource(javaCode); - - return parser; - } - - private char[] readFileToCharArray(FileReader fileReader) throws IOException { - StringBuilder fileData = new StringBuilder(); - BufferedReader br = new BufferedReader(fileReader); - - char[] buf = new char[10]; - int numRead = 0; - while ((numRead = br.read(buf)) != -1) { - String readData = String.valueOf(buf, 0, numRead); - fileData.append(readData); - buf = new char[1024]; - } - br.close(); - - return fileData.toString().toCharArray(); - } -} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternHunkParser.java similarity index 92% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java rename to src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternHunkParser.java index 4d885ea..eaf7e7c 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternHunkParser.java @@ -1,6 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser.violations; - -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +package edu.lu.uni.serval.FixPatternParser; import java.io.*; import java.util.List; @@ -13,7 +11,7 @@ import java.util.List; * @author kui.liu * */ -public class FixedViolationHunkParser extends FixedViolationParser { +public class FixedPatternHunkParser extends FixedPatternParser { public String testingInfo = ""; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternParser.java similarity index 86% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java rename to src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternParser.java index 3bbc4eb..ae3255a 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternParser.java @@ -1,10 +1,7 @@ -package edu.lu.uni.serval.FixPatternParser.violations; +package edu.lu.uni.serval.FixPatternParser; import com.github.gumtreediff.actions.model.Action; -import edu.lu.uni.serval.FixPatternParser.Parser; import edu.lu.uni.serval.gumtree.GumTreeComparer; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; import edu.lu.uni.serval.utils.ListSorter; import java.io.File; @@ -17,7 +14,7 @@ import java.util.List; * @author kui.liu * */ -public class FixedViolationParser extends Parser { +public class FixedPatternParser extends Parser { /* * ResultType: diff --git a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalActionSet.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalActionSet.java similarity index 99% rename from src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalActionSet.java rename to src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalActionSet.java index 4b5fb60..a0125b6 100644 --- a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalActionSet.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalActionSet.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.gumtree.regroup; +package edu.lu.uni.serval.FixPatternParser; import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.tree.ITree; diff --git a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalRegrouper.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java rename to src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalRegrouper.java index db1e1db..004c773 100644 --- a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalRegrouper.java @@ -1,8 +1,8 @@ -package edu.lu.uni.serval.gumtree.regroup; +package edu.lu.uni.serval.FixPatternParser; import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.tree.ITree; -import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap; +import edu.lu.uni.serval.utils.ASTNodeMap; import edu.lu.uni.serval.utils.ListSorter; import java.util.ArrayList; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java index d4cf22b..9c8a4f8 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java @@ -1,6 +1,7 @@ package edu.lu.uni.serval.fixminer; -import edu.lu.uni.serval.fixminer.cluster.*; +import edu.lu.uni.serval.fixminer.jobs.*; +import edu.lu.uni.serval.utils.CallShell; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java similarity index 99% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoader.java rename to src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java index e4e0441..67fed8f 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer; import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.model.Action; @@ -6,8 +6,8 @@ import com.github.gumtreediff.matchers.Matcher; import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.Tree; -import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.utils.ASTNodeMap; +import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.*; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java index 44575a7..be93c0b 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java @@ -2,9 +2,9 @@ package edu.lu.uni.serval.fixminer; import akka.actor.ActorRef; import akka.actor.ActorSystem; -import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; -import edu.lu.uni.serval.MultipleThreadsParser.ParseFixPatternActor; -import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage; +import edu.lu.uni.serval.fixminer.cluster.akka.MessageFile; +import edu.lu.uni.serval.fixminer.cluster.akka.EDiffActor; +import edu.lu.uni.serval.fixminer.cluster.akka.EDiffMessage; import edu.lu.uni.serval.utils.FileHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,11 +96,11 @@ public class TestHunkParserSingleFile { ActorSystem system = null; ActorRef parsingActor = null; - final WorkMessage msg = new WorkMessage(0, msgFiles); + final EDiffMessage msg = new EDiffMessage(0, msgFiles); try { log.info("Akka begins..."); system = ActorSystem.create("Mining-FixPattern-System"); - parsingActor = system.actorOf(ParseFixPatternActor.props(1, "dataset"), "mine-fix-pattern-actor"); + parsingActor = system.actorOf(EDiffActor.props(1, "dataset"), "mine-fix-pattern-actor"); parsingActor.tell(msg, ActorRef.noSender()); } catch (Exception e) { system.shutdown(); @@ -109,7 +109,7 @@ public class TestHunkParserSingleFile { // int counter = 0; // for (MessageFile msgFile : msgFiles) { -// FixedViolationHunkParser parser = new FixedViolationHunkParser(); +// FixedPatternHunkParser parser = new FixedPatternHunkParser(); // // final ExecutorService executor = Executors.newSingleThreadExecutor(); // // schedule the work diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java index 49c03af..cdfcc45 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java @@ -5,7 +5,7 @@ import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.matchers.Matcher; import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; -import edu.lu.uni.serval.FixPattern.utils.EDiff; +import edu.lu.uni.serval.utils.EDiff; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/WorkMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/WorkMessage.java deleted file mode 100644 index f009b25..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/WorkMessage.java +++ /dev/null @@ -1,51 +0,0 @@ -package edu.lu.uni.serval.fixminer.cluster; - -import java.util.List; - -public class WorkMessage { - - private int id; - private List msgFiles; - - public String getDbDir() { - return dbDir; - } - - public String getServerWait() { - return serverWait; - } - - private String dbDir; - private String serverWait; - - public String getInputPath() { - return inputPath; - } - - private String inputPath; - - public String getInnerPort() { - return innerPort; - } - - private String innerPort; - - public WorkMessage(int id, List msgFiles,String innerPort,String inputPath,String dbDir,String serverWait) { - super(); - this.id = id; - this.msgFiles = msgFiles; - this.innerPort = innerPort; - this.inputPath = inputPath; - this.dbDir = dbDir; - this.serverWait = serverWait; - } - - public int getId() { - return id; - } - - public List getMsgFiles() { - return msgFiles; - } - -} \ No newline at end of file diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java index d76b532..c504528 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java @@ -2,7 +2,6 @@ package edu.lu.uni.serval.fixminer.cluster.akka; import akka.actor.ActorRef; import akka.actor.ActorSystem; -import edu.lu.uni.serval.fixminer.cluster.TreeActor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternActor.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffActor.java similarity index 70% rename from src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternActor.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffActor.java index 46ea6df..42be0b0 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffActor.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.MultipleThreadsParser; +package edu.lu.uni.serval.fixminer.cluster.akka; import akka.actor.ActorRef; import akka.actor.Props; @@ -10,29 +10,29 @@ import org.slf4j.LoggerFactory; import java.util.List; -public class ParseFixPatternActor extends UntypedActor { +public class EDiffActor extends UntypedActor { - private static Logger logger = LoggerFactory.getLogger(ParseFixPatternActor.class); + private static Logger logger = LoggerFactory.getLogger(EDiffActor.class); private ActorRef mineRouter; private final int numberOfWorkers; private int counter = 0; - public ParseFixPatternActor(int numberOfWorkers, String project) { + public EDiffActor(int numberOfWorkers, String project) { mineRouter = this.getContext().actorOf(new RoundRobinPool(numberOfWorkers) - .props(ParseFixPatternWorker.props(project)), "mine-fix-pattern-router"); + .props(EDiffWorker.props(project)), "mine-fix-pattern-router"); this.numberOfWorkers = numberOfWorkers; } public static Props props(final int numberOfWorkers, final String project) { - return Props.create(new Creator() { + return Props.create(new Creator() { private static final long serialVersionUID = 9207427376110704705L; @Override - public ParseFixPatternActor create() throws Exception { - return new ParseFixPatternActor(numberOfWorkers, project); + public EDiffActor create() throws Exception { + return new EDiffActor(numberOfWorkers, project); } }); @@ -41,8 +41,8 @@ public class ParseFixPatternActor extends UntypedActor { @SuppressWarnings("deprecation") @Override public void onReceive(Object message) throws Exception { - if (message instanceof WorkMessage) { - List files = ((WorkMessage) message).getMsgFiles(); + if (message instanceof EDiffMessage) { + List files = ((EDiffMessage) message).getMsgFiles(); int size = files.size(); int average = size / numberOfWorkers; int reminder = size % numberOfWorkers; @@ -54,7 +54,7 @@ public class ParseFixPatternActor extends UntypedActor { int toIndex = (i + 1) * average + counter; List filesOfWorkers = files.subList(fromIndex, toIndex); - final WorkMessage workMsg = new WorkMessage(i + 1, filesOfWorkers); + final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers); mineRouter.tell(workMsg, getSelf()); logger.info("Assign a task to worker #" + (i + 1) + "..."); } diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/WorkMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffMessage.java similarity index 64% rename from src/main/java/edu/lu/uni/serval/MultipleThreadsParser/WorkMessage.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffMessage.java index 9d30eb6..afea756 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/WorkMessage.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffMessage.java @@ -1,13 +1,13 @@ -package edu.lu.uni.serval.MultipleThreadsParser; +package edu.lu.uni.serval.fixminer.cluster.akka; import java.util.List; -public class WorkMessage { +public class EDiffMessage { private int id; private List msgFiles; - public WorkMessage(int id, List msgFiles) { + public EDiffMessage(int id, List msgFiles) { super(); this.id = id; this.msgFiles = msgFiles; diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffWorker.java similarity index 85% rename from src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffWorker.java index 3084dbe..5c8a77f 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffWorker.java @@ -1,10 +1,10 @@ -package edu.lu.uni.serval.MultipleThreadsParser; +package edu.lu.uni.serval.fixminer.cluster.akka; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; +import edu.lu.uni.serval.FixPatternParser.FixedPatternHunkParser; import edu.lu.uni.serval.FixPatternParser.RunnableParser; -import edu.lu.uni.serval.FixPatternParser.violations.FixedViolationHunkParser; import edu.lu.uni.serval.config.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,24 +13,24 @@ import java.io.File; import java.util.List; import java.util.concurrent.*; -public class ParseFixPatternWorker extends UntypedActor { - private static Logger log = LoggerFactory.getLogger(ParseFixPatternActor.class); +public class EDiffWorker extends UntypedActor { + private static Logger log = LoggerFactory.getLogger(EDiffActor.class); private String project; - public ParseFixPatternWorker(String project) { + public EDiffWorker(String project) { this.project = project; } public static Props props(final String project) { - return Props.create(new Creator() { + return Props.create(new Creator() { private static final long serialVersionUID = -7615153844097275009L; @Override - public ParseFixPatternWorker create() throws Exception { - return new ParseFixPatternWorker(project); + public EDiffWorker create() throws Exception { + return new EDiffWorker(project); } }); @@ -38,8 +38,8 @@ public class ParseFixPatternWorker extends UntypedActor { @Override public void onReceive(Object message) throws Exception { - if (message instanceof WorkMessage) { - WorkMessage msg = (WorkMessage) message; + if (message instanceof EDiffMessage) { + EDiffMessage msg = (EDiffMessage) message; List files = msg.getMsgFiles(); StringBuilder editScripts = new StringBuilder(); StringBuilder patchesSourceCode = new StringBuilder(); @@ -68,7 +68,7 @@ public class ParseFixPatternWorker extends UntypedActor { File diffentryFile = msgFile.getDiffEntryFile(); - FixedViolationHunkParser parser = new FixedViolationHunkParser(); + FixedPatternHunkParser parser = new FixedPatternHunkParser(); final ExecutorService executor = Executors.newSingleThreadExecutor(); // schedule the work diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/MessageFile.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/MessageFile.java similarity index 93% rename from src/main/java/edu/lu/uni/serval/MultipleThreadsParser/MessageFile.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/MessageFile.java index de224ef..e2bea96 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/MessageFile.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/MessageFile.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.MultipleThreadsParser; +package edu.lu.uni.serval.fixminer.cluster.akka; import java.io.File; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeActor.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeActor.java similarity index 95% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeActor.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeActor.java index 939c129..6957275 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeActor.java @@ -1,11 +1,10 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.cluster.akka; import akka.actor.ActorRef; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; import akka.routing.RoundRobinPool; -import edu.lu.uni.serval.fixminer.cluster.akka.TreeMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.JedisPool; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeWorker.java similarity index 96% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeWorker.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeWorker.java index cfc7bb9..5eb61e1 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeWorker.java @@ -1,10 +1,11 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.cluster.akka; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; import edu.lu.uni.serval.config.Configuration; -import edu.lu.uni.serval.fixminer.cluster.akka.TreeMessage; +import edu.lu.uni.serval.fixminer.cluster.Compare; +import edu.lu.uni.serval.fixminer.cluster.RunnableCompare; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.JedisPool; @@ -65,7 +66,7 @@ public class TreeWorker extends UntypedActor { if(message instanceof TreeMessage) { -// if (message instanceof edu.lu.uni.serval.MultipleThreadsParser.WorkMessage) { +// if (message instanceof edu.lu.uni.serval.fixminer.cluster.akka.EDiffMessage) { TreeMessage msg = (TreeMessage) message; List files = msg.getName(); JedisPool innerPool = msg.getInnerPool(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java similarity index 95% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/AkkaTreeLoader.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java index 2cc115b..439c845 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/AkkaTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java @@ -1,6 +1,7 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.jobs; -import edu.lu.uni.serval.FixPattern.utils.PoolBuilder; +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.PoolBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java similarity index 97% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java index 87ab887..dfaa3c0 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java @@ -1,6 +1,7 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.jobs; -import edu.lu.uni.serval.FixPattern.utils.PoolBuilder; +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.PoolBuilder; import edu.lu.uni.serval.utils.FileHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java similarity index 87% rename from src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java index 43895e0..0ec8660 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java @@ -1,10 +1,10 @@ -package edu.lu.uni.serval.fixminer; +package edu.lu.uni.serval.fixminer.jobs; import akka.actor.ActorRef; import akka.actor.ActorSystem; -import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; -import edu.lu.uni.serval.MultipleThreadsParser.ParseFixPatternActor; -import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage; +import edu.lu.uni.serval.fixminer.cluster.akka.EDiffMessage; +import edu.lu.uni.serval.fixminer.cluster.akka.MessageFile; +import edu.lu.uni.serval.fixminer.cluster.akka.EDiffActor; import edu.lu.uni.serval.utils.FileHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,12 +56,12 @@ public class EnhancedASTDiff { ActorSystem system = null; ActorRef parsingActor = null; - final WorkMessage msg = new WorkMessage(0, msgFiles); + final EDiffMessage msg = new EDiffMessage(0, msgFiles); try { log.info("Akka begins..."); system = ActorSystem.create("Mining-FixPattern-System"); - parsingActor = system.actorOf(ParseFixPatternActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor"); + parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor"); parsingActor.tell(msg, ActorRef.noSender()); } catch (Exception e) { system.shutdown(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/ImportPairs2DB.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java similarity index 95% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/ImportPairs2DB.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java index 6249a4a..bf2befb 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/ImportPairs2DB.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java @@ -1,5 +1,6 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.jobs; +import edu.lu.uni.serval.utils.CallShell; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java similarity index 97% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java index fa6ed79..723babf 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.jobs; import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.model.Action; @@ -6,10 +6,11 @@ import com.github.gumtreediff.matchers.Matcher; import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.FixPattern.utils.EDiff; -import edu.lu.uni.serval.FixPattern.utils.EDiffHelper; -import edu.lu.uni.serval.FixPattern.utils.PoolBuilder; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.EDiff; +import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.PoolBuilder; +import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import edu.lu.uni.serval.utils.FileHelper; import org.javatuples.Pair; import org.slf4j.Logger; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster3.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java similarity index 99% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster3.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java index 661595a..73562b8 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster3.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java @@ -1,10 +1,11 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.jobs; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.FixPattern.utils.EDiffHelper; -import edu.lu.uni.serval.FixPattern.utils.PoolBuilder; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.PoolBuilder; +import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import edu.lu.uni.serval.utils.FileHelper; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.similarity.JaroWinklerDistance; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/PatternExtractor.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java similarity index 99% rename from src/main/java/edu/lu/uni/serval/fixminer/PatternExtractor.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java index bbc33a5..0dd8b6e 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/PatternExtractor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java @@ -1,7 +1,7 @@ -package edu.lu.uni.serval.fixminer; +package edu.lu.uni.serval.fixminer.jobs; -import edu.lu.uni.serval.FixPattern.utils.Checker; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.utils.Checker; +import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java similarity index 95% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java index 39bba51..9e5541d 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java @@ -1,8 +1,9 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.jobs; -import edu.lu.uni.serval.FixPattern.utils.EDiffHelper; -import edu.lu.uni.serval.FixPattern.utils.PoolBuilder; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.PoolBuilder; +import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeLoaderClusterL1.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java similarity index 96% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeLoaderClusterL1.java rename to src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java index 36a480a..99a32fe 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeLoaderClusterL1.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java @@ -1,6 +1,7 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.jobs; -import edu.lu.uni.serval.FixPattern.utils.PoolBuilder; +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.PoolBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/FixPattern/utils/ASTNodeMap.java b/src/main/java/edu/lu/uni/serval/utils/ASTNodeMap.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/FixPattern/utils/ASTNodeMap.java rename to src/main/java/edu/lu/uni/serval/utils/ASTNodeMap.java index 266063c..c2d5bf8 100644 --- a/src/main/java/edu/lu/uni/serval/FixPattern/utils/ASTNodeMap.java +++ b/src/main/java/edu/lu/uni/serval/utils/ASTNodeMap.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPattern.utils; +package edu.lu.uni.serval.utils; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java b/src/main/java/edu/lu/uni/serval/utils/CallShell.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java rename to src/main/java/edu/lu/uni/serval/utils/CallShell.java index 9c1d05d..9e4b56d 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java +++ b/src/main/java/edu/lu/uni/serval/utils/CallShell.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.utils; /** * Created by anilkoyuncu on 17/04/2018. diff --git a/src/main/java/edu/lu/uni/serval/FixPattern/utils/Checker.java b/src/main/java/edu/lu/uni/serval/utils/Checker.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/FixPattern/utils/Checker.java rename to src/main/java/edu/lu/uni/serval/utils/Checker.java index 83e1e3a..89752fa 100644 --- a/src/main/java/edu/lu/uni/serval/FixPattern/utils/Checker.java +++ b/src/main/java/edu/lu/uni/serval/utils/Checker.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPattern.utils; +package edu.lu.uni.serval.utils; public class Checker { diff --git a/src/main/java/edu/lu/uni/serval/FixPattern/utils/EDiff.java b/src/main/java/edu/lu/uni/serval/utils/EDiff.java similarity index 96% rename from src/main/java/edu/lu/uni/serval/FixPattern/utils/EDiff.java rename to src/main/java/edu/lu/uni/serval/utils/EDiff.java index acec7ff..a2a1bfc 100644 --- a/src/main/java/edu/lu/uni/serval/FixPattern/utils/EDiff.java +++ b/src/main/java/edu/lu/uni/serval/utils/EDiff.java @@ -1,9 +1,9 @@ -package edu.lu.uni.serval.FixPattern.utils; +package edu.lu.uni.serval.utils; import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/FixPattern/utils/EDiffHelper.java b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java similarity index 96% rename from src/main/java/edu/lu/uni/serval/FixPattern/utils/EDiffHelper.java rename to src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java index b6f7980..ae7d3f0 100644 --- a/src/main/java/edu/lu/uni/serval/FixPattern/utils/EDiffHelper.java +++ b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java @@ -1,8 +1,8 @@ -package edu.lu.uni.serval.FixPattern.utils; +package edu.lu.uni.serval.utils; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/edu/lu/uni/serval/FixPattern/utils/PoolBuilder.java b/src/main/java/edu/lu/uni/serval/utils/PoolBuilder.java similarity index 95% rename from src/main/java/edu/lu/uni/serval/FixPattern/utils/PoolBuilder.java rename to src/main/java/edu/lu/uni/serval/utils/PoolBuilder.java index fba3f04..31562f2 100644 --- a/src/main/java/edu/lu/uni/serval/FixPattern/utils/PoolBuilder.java +++ b/src/main/java/edu/lu/uni/serval/utils/PoolBuilder.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPattern.utils; +package edu.lu.uni.serval.utils; import redis.clients.jedis.JedisPoolConfig; From b76fd006958094cce4181cd930163d7b857611d2 Mon Sep 17 00:00:00 2001 From: fixminer Date: Tue, 18 Sep 2018 14:59:44 +0200 Subject: [PATCH 2/4] rc 4 --- .../edu/lu/uni/serval/fixminer/Launcher.java | 51 +-- .../fixminer/MultiThreadTreeLoader.java | 2 +- .../fixminer/TestHunkParserSingleFile.java | 298 +++++++++--------- .../uni/serval/fixminer/akka/BaseMessage.java | 46 +++ .../akka => akka/compare}/AkkaTreeParser.java | 10 +- .../{cluster => akka/compare}/Compare.java | 6 +- .../compare}/RunnableCompare.java | 2 +- .../akka => akka/compare}/TreeActor.java | 4 +- .../akka => akka/compare}/TreeMessage.java | 23 +- .../akka => akka/compare}/TreeWorker.java | 15 +- .../akka => akka/ediff}/EDiffActor.java | 8 +- .../akka/ediff/EDiffHunkParser.java} | 70 ++-- .../fixminer/akka/ediff/EDiffMessage.java | 45 +++ .../akka/ediff/EDiffParser.java} | 21 +- .../akka => akka/ediff}/EDiffWorker.java | 20 +- .../akka/ediff}/HierarchicalActionSet.java | 2 +- .../akka/ediff}/HierarchicalRegrouper.java | 2 +- .../akka => akka/ediff}/MessageFile.java | 2 +- .../akka/ediff}/Parser.java | 4 +- .../akka/ediff}/ParserInterface.java | 2 +- .../akka/ediff}/RunnableParser.java | 8 +- .../fixminer/cluster/akka/EDiffMessage.java | 24 -- .../serval/fixminer/jobs/AkkaTreeLoader.java | 16 +- .../serval/fixminer/jobs/CalculatePairs.java | 86 +++-- .../serval/fixminer/jobs/EnhancedASTDiff.java | 21 +- .../serval/fixminer/jobs/ImportPairs2DB.java | 2 +- .../jobs/MultiThreadTreeLoaderCluster.java | 21 +- .../jobs/MultiThreadTreeLoaderCluster3.java | 19 +- .../fixminer/jobs/PatternExtractor.java | 2 +- .../fixminer/jobs/StoreEDiffInCache.java | 6 +- .../fixminer/jobs/TreeLoaderClusterL1.java | 4 +- .../edu/lu/uni/serval/utils/CallShell.java | 6 + .../java/edu/lu/uni/serval/utils/EDiff.java | 2 +- .../edu/lu/uni/serval/utils/EDiffHelper.java | 43 ++- 34 files changed, 527 insertions(+), 366 deletions(-) create mode 100644 src/main/java/edu/lu/uni/serval/fixminer/akka/BaseMessage.java rename src/main/java/edu/lu/uni/serval/fixminer/{cluster/akka => akka/compare}/AkkaTreeParser.java (90%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster => akka/compare}/Compare.java (94%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster => akka/compare}/RunnableCompare.java (90%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster/akka => akka/compare}/TreeActor.java (94%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster/akka => akka/compare}/TreeMessage.java (64%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster/akka => akka/compare}/TreeWorker.java (93%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster/akka => akka/ediff}/EDiffActor.java (88%) rename src/main/java/edu/lu/uni/serval/{FixPatternParser/FixedPatternHunkParser.java => fixminer/akka/ediff/EDiffHunkParser.java} (55%) create mode 100644 src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java rename src/main/java/edu/lu/uni/serval/{FixPatternParser/FixedPatternParser.java => fixminer/akka/ediff/EDiffParser.java} (79%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster/akka => akka/ediff}/EDiffWorker.java (83%) rename src/main/java/edu/lu/uni/serval/{FixPatternParser => fixminer/akka/ediff}/HierarchicalActionSet.java (98%) rename src/main/java/edu/lu/uni/serval/{FixPatternParser => fixminer/akka/ediff}/HierarchicalRegrouper.java (99%) rename src/main/java/edu/lu/uni/serval/fixminer/{cluster/akka => akka/ediff}/MessageFile.java (93%) rename src/main/java/edu/lu/uni/serval/{FixPatternParser => fixminer/akka/ediff}/Parser.java (92%) rename src/main/java/edu/lu/uni/serval/{FixPatternParser => fixminer/akka/ediff}/ParserInterface.java (88%) rename src/main/java/edu/lu/uni/serval/{FixPatternParser => fixminer/akka/ediff}/RunnableParser.java (80%) delete mode 100644 src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffMessage.java diff --git a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java index 9c8a4f8..2dec952 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java @@ -27,34 +27,35 @@ public class Launcher { appProps.load(new FileInputStream(appConfigPath)); String portInner = appProps.getProperty("portInner","6380"); -// String serverWait = appProps.getProperty("serverWait", "50000"); String numOfWorkers = appProps.getProperty("numOfWorkers", "10"); String jobType = appProps.getProperty("jobType","ALL"); String portDumps = appProps.getProperty("portDumps","6399"); String pythonPath = appProps.getProperty("pythonPath","/Users/anilkoyuncu/bugStudy/code/python"); String datasetPath = appProps.getProperty("datasetPath","/Users/anilkoyuncu/bugStudy/dataset"); String pjName = appProps.getProperty("pjName","allDataset"); -// String dbNo = appProps.getProperty("dbNo","0"); String actionType = appProps.getProperty("actionType","ALL"); String threshold = appProps.getProperty("threshold","1"); String cursor = appProps.getProperty("cursor","10000000"); String chunk = appProps.getProperty("chunk","1.txt"); + String eDiffTimeout = appProps.getProperty("eDiffTimeout","900"); + String isBig = appProps.getProperty("isBigPair","true"); + boolean isBigPair = Boolean.parseBoolean(isBig); String parameters = String.format("\nportInner %s " + "\nnumOfWorkers %s " + "\njobType %s \nport %s " + "\npythonPath %s \ndatasetPath %s" + - "\npjName %s \nactionType %s \nthreshold %s \ncursor %s" - , portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor); + "\npjName %s \nactionType %s \nthreshold %s \ncursor %s \neDiffTimeout %s \nisBigPair %s" + , portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,eDiffTimeout,isBig); log.info(parameters); - mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk); + mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk,eDiffTimeout,isBigPair); } - public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk){ + public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk, String eDiffTimeout, boolean isBigPair){ String dbDir; @@ -68,33 +69,35 @@ public class Launcher { dbDir = datasetPath + "/redis"; pairsPath = datasetPath + "/pairsImport"+pjName; dumpsName = "dumps-"+pjName+".rdb"; + int iCursor = Integer.valueOf(cursor); try { switch (jobType) { case "ENHANCEDASTDIFF": - EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName); + EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName, eDiffTimeout,actionType); break; case "CACHE": StoreEDiffInCache.main(gumOutput, portDumps, dbDir, actionType+dumpsName,actionType); break; case "SI": - CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType); + CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType,isBigPair,iCursor); ImportPairs2DB.main(pairsPath+actionType, portInner, dbDir,datasetPath); break; case "SIMI": - AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,cursor,chunk); + AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,iCursor,chunk,eDiffTimeout); break; case "LEVEL1": - level1(portInner, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput); + String indexFile = pairsPath+actionType + "/" +pjName+actionType+".index"; + level1(portInner, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, gumInput,iCursor,indexFile); break; //CALC python abstractPatch.py to from cluster folder case "LEVEL2": - level2(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); + level2(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput,iCursor); break; //CALC via python case "LEVEL3": - level3(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); + level3(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput,iCursor); break; case "EXTRACTPATTERN": PatternExtractor.mainLaunch(portInner,numOfWorkers,jobType,portDumps,pythonPath,datasetPath,pjName,actionType,threshold); @@ -112,17 +115,17 @@ public class Launcher { } - private static void level1(String portInner, String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String pairsPath, String dumpsName, String gumInput) throws Exception { + private static void level1(String portInner, String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String gumInput,int cursor,String indexFile) throws Exception { - TreeLoaderClusterL1.main(portInner, port, dbDir, "level1-"+pjName+ actionType+".rdb", dbDir ,pjName + actionType); + TreeLoaderClusterL1.main(portInner, port, dbDir, "level1-"+pjName+ actionType+".rdb", dbDir ,pjName + actionType,cursor); CallShell cs1 =new CallShell(); String db1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String db11 = String.format(db1, dbDir,"level1-"+pjName+ actionType+".rdb" ,Integer.valueOf(port)); cs1.runShell(db11, port); - String runpy = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s"; - String formatRunPy = String.format(runpy,pythonPath +"/abstractPatch.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, "matches" + pjName + actionType, threshold); + String runpy = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s %s"; + String formatRunPy = String.format(runpy,pythonPath +"/abstractPatch.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, "matches" + pjName + actionType, threshold,indexFile); cs1.runShell(formatRunPy); String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; @@ -130,18 +133,18 @@ public class Launcher { cs1.runShell(stopServer, port); } - private static void level2(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput) throws Exception { + private static void level2(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput,int cursor) throws Exception { String stopServer; MultiThreadTreeLoaderCluster.calculatePairsOfClusters(datasetPath + "/cluster"+pjName+ actionType, datasetPath,actionType); - MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath+"/pairs"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType); + MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath+"/pairs"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType,cursor); CallShell cs3 =new CallShell(); String db22 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String db1b = String.format(db22, dbDir,"clusterl1-"+pjName+actionType+".rdb",Integer.valueOf(port)); cs3.runShell(db1b, port); - String runpy2 = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s"; - String formatRunPy1a = String.format(runpy2,pythonPath +"/abstractPatchCluster.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold); + String runpy2 = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s %s"; + String formatRunPy1a = String.format(runpy2,pythonPath +"/abstractPatchCluster.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold,"dummy"); cs3.runShell(formatRunPy1a); String stopServer1a = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; @@ -149,18 +152,18 @@ public class Launcher { cs3.runShell(stopServer, port); } - private static void level3(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput) throws Exception { + private static void level3(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput,int cursor) throws Exception { String stopServer; MultiThreadTreeLoaderCluster3.calculatePairsOfClusters(datasetPath + "/cluster-2l"+pjName+ actionType, datasetPath,actionType); - MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath+"/pairs-2l"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType); + MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath+"/pairs-2l"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType,cursor); CallShell cs5 =new CallShell(); String dba = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String dbaa = String.format(dba, dbDir,"clusterl2-"+pjName+actionType+".rdb",Integer.valueOf(port)); cs5.runShell(dbaa, port); - String runpya = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s"; - String formatRunPya = String.format(runpya,pythonPath +"/abstractPatchClusterLevel3.py", gumInput, datasetPath + "/cluster-3l"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold); + String runpya = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s %s"; + String formatRunPya = String.format(runpya,pythonPath +"/abstractPatchClusterLevel3.py", gumInput, datasetPath + "/cluster-3l"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold,"dummy"); cs5.runShell(formatRunPya); String stopServera = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java index 67fed8f..8734393 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java @@ -6,8 +6,8 @@ import com.github.gumtreediff.matchers.Matcher; import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.Tree; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import edu.lu.uni.serval.utils.ASTNodeMap; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.*; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java index be93c0b..f116deb 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java @@ -2,9 +2,9 @@ package edu.lu.uni.serval.fixminer; import akka.actor.ActorRef; import akka.actor.ActorSystem; -import edu.lu.uni.serval.fixminer.cluster.akka.MessageFile; -import edu.lu.uni.serval.fixminer.cluster.akka.EDiffActor; -import edu.lu.uni.serval.fixminer.cluster.akka.EDiffMessage; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage; +import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile; import edu.lu.uni.serval.utils.FileHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,155 +19,155 @@ import java.util.stream.Stream; public class TestHunkParserSingleFile { private static Logger log = LoggerFactory.getLogger(TestHunkParserSingleFile.class); - public static void main(String[] args) { - // input data - -// String rootPath = "/Users/anilkoyuncu/bugStudy"; - String inputPath; - String outputPath; - if(args.length > 0){ - inputPath = args[1]; - outputPath = args[0]; - }else{ -// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeInputBug4"; - inputPath = "/Users/anilkoyuncu/bugStudy/dataset/allDataset"; -// outputPath = "/Users/anilkoyuncu/bugStudy/code/python/GumTreeOutput2/"; - outputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputSingle"; - } - - //5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.txt_1_CAMEL - - File folder = new File(inputPath); - File[] listOfFiles = folder.listFiles(); - Stream stream = Arrays.stream(listOfFiles); - List folders = stream - .filter(x -> !x.getName().startsWith(".")) - - .collect(Collectors.toList()); - -// List targetList = new ArrayList(); -// for (File f:folders){ -// for(File f1 :f.listFiles()){ -// if (!f1.getName().startsWith(".")){ -// targetList.add(f1); -// } -// } +// public static void main(String[] args) { +// // input data +// +//// String rootPath = "/Users/anilkoyuncu/bugStudy"; +// String inputPath; +// String outputPath; +// if(args.length > 0){ +// inputPath = args[1]; +// outputPath = args[0]; +// }else{ +//// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeInputBug4"; +// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/allDataset"; +//// outputPath = "/Users/anilkoyuncu/bugStudy/code/python/GumTreeOutput2/"; +// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputSingle"; // } - -// List pjList = Arrays.asList("DATAJPA","ZXing","PDE","SWS","SWT", "SWF", "COLLECTIONS", "JDT"); - List files = new ArrayList(); -// files.add("5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.java"); - files.add("d6c5e5_9f96d9_hbase-server#src#main#java#org#apache#hadoop#hbase#master#RegionStates.java"); - for(String f : files){ - String pjName = "HBASE"; -// for (File target : folders) { -// String pjName = target.getName(); -// if (!pjList.contains(pjName)){ -// continue; -// } - -// final List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" - MessageFile messageFile = getMessageFile(inputPath + "/" + pjName +"/", f); - - List msgFiles = new ArrayList<>(); - msgFiles.add(messageFile); - String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; - final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list"; - final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list"; - final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list"; - final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv"; - final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list"; - - - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ActionSetDumps"); - FileHelper.deleteDirectory(editScriptsFilePath); - FileHelper.deleteDirectory(patchesSourceCodeFilePath); - FileHelper.deleteDirectory(buggyTokensFilePath); - FileHelper.deleteDirectory(editScriptSizesFilePath); - FileHelper.deleteDirectory(alarmTypesFilePath); - - StringBuilder astEditScripts = new StringBuilder(); - StringBuilder tokens = new StringBuilder(); - StringBuilder sizes = new StringBuilder(); - StringBuilder patches = new StringBuilder(); - StringBuilder alarmTypes = new StringBuilder(); - -// int a = 0; - - ActorSystem system = null; - ActorRef parsingActor = null; - final EDiffMessage msg = new EDiffMessage(0, msgFiles); - try { - log.info("Akka begins..."); - system = ActorSystem.create("Mining-FixPattern-System"); - parsingActor = system.actorOf(EDiffActor.props(1, "dataset"), "mine-fix-pattern-actor"); - parsingActor.tell(msg, ActorRef.noSender()); - } catch (Exception e) { - system.shutdown(); - e.printStackTrace(); - } - -// int counter = 0; -// for (MessageFile msgFile : msgFiles) { -// FixedPatternHunkParser parser = new FixedPatternHunkParser(); // -// final ExecutorService executor = Executors.newSingleThreadExecutor(); -// // schedule the work -// final Future future = executor.submit(new RunnableParser(msgFile.getPrevFile(), -// msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser)); -// try { -// // where we wait for task to complete -// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); -// String editScripts = parser.getAstEditScripts(); -// if (!editScripts.equals("")) { -// astEditScripts.append(editScripts); -// tokens.append(parser.getTokensOfSourceCode()); -// sizes.append(parser.getSizes()); -// patches.append(parser.getPatchesSourceCode()); -// alarmTypes.append(parser.getAlarmTypes()); +// //5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.txt_1_CAMEL // -// a++; -// if (a % 100 == 0) { -// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); -// FileHelper.outputToFile(buggyTokensFilePath, tokens, true); -// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); -// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); -// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); -// astEditScripts.setLength(0); -// tokens.setLength(0); -// sizes.setLength(0); -// patches.setLength(0); -// alarmTypes.setLength(0); -// System.out.println("Finish of parsing " + a + " files......"); -// } -// } -// } catch (TimeoutException e) { -// err.println("task timed out"); -// future.cancel(true /* mayInterruptIfRunning */); -// } catch (InterruptedException e) { -// err.println("task interrupted"); -// } catch (ExecutionException e) { -// err.println("task aborted"); -// } finally { -// executor.shutdownNow(); -// } -// } - - FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); - FileHelper.outputToFile(buggyTokensFilePath, tokens, true); - FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); - FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); - FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); - astEditScripts.setLength(0); - tokens.setLength(0); - sizes.setLength(0); - patches.setLength(0); - alarmTypes.setLength(0); - - -// classifyByAlarmTypes(); - } - } +// File folder = new File(inputPath); +// File[] listOfFiles = folder.listFiles(); +// Stream stream = Arrays.stream(listOfFiles); +// List folders = stream +// .filter(x -> !x.getName().startsWith(".")) +// +// .collect(Collectors.toList()); +// +//// List targetList = new ArrayList(); +//// for (File f:folders){ +//// for(File f1 :f.listFiles()){ +//// if (!f1.getName().startsWith(".")){ +//// targetList.add(f1); +//// } +//// } +//// } +// +//// List pjList = Arrays.asList("DATAJPA","ZXing","PDE","SWS","SWT", "SWF", "COLLECTIONS", "JDT"); +// List files = new ArrayList(); +//// files.add("5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.java"); +// files.add("d6c5e5_9f96d9_hbase-server#src#main#java#org#apache#hadoop#hbase#master#RegionStates.java"); +// for(String f : files){ +// String pjName = "HBASE"; +//// for (File target : folders) { +//// String pjName = target.getName(); +//// if (!pjList.contains(pjName)){ +//// continue; +//// } +// +//// final List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" +// MessageFile messageFile = getMessageFile(inputPath + "/" + pjName +"/", f); +// +// List msgFiles = new ArrayList<>(); +// msgFiles.add(messageFile); +// String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; +// final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list"; +// final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list"; +// final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list"; +// final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv"; +// final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list"; +// +// +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ActionSetDumps"); +// FileHelper.deleteDirectory(editScriptsFilePath); +// FileHelper.deleteDirectory(patchesSourceCodeFilePath); +// FileHelper.deleteDirectory(buggyTokensFilePath); +// FileHelper.deleteDirectory(editScriptSizesFilePath); +// FileHelper.deleteDirectory(alarmTypesFilePath); +// +// StringBuilder astEditScripts = new StringBuilder(); +// StringBuilder tokens = new StringBuilder(); +// StringBuilder sizes = new StringBuilder(); +// StringBuilder patches = new StringBuilder(); +// StringBuilder alarmTypes = new StringBuilder(); +// +//// int a = 0; +// +// ActorSystem system = null; +// ActorRef parsingActor = null; +// final EDiffMessage msg = new EDiffMessage(0, msgFiles); +// try { +// log.info("Akka begins..."); +// system = ActorSystem.create("Mining-FixPattern-System"); +// parsingActor = system.actorOf(EDiffActor.props(1, "dataset"), "mine-fix-pattern-actor"); +// parsingActor.tell(msg, ActorRef.noSender()); +// } catch (Exception e) { +// system.shutdown(); +// e.printStackTrace(); +// } +// +//// int counter = 0; +//// for (MessageFile msgFile : msgFiles) { +//// EDiffHunkParser parser = new EDiffHunkParser(); +//// +//// final ExecutorService executor = Executors.newSingleThreadExecutor(); +//// // schedule the work +//// final Future future = executor.submit(new RunnableParser(msgFile.getPrevFile(), +//// msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser)); +//// try { +//// // where we wait for task to complete +//// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); +//// String editScripts = parser.getAstEditScripts(); +//// if (!editScripts.equals("")) { +//// astEditScripts.append(editScripts); +//// tokens.append(parser.getTokensOfSourceCode()); +//// sizes.append(parser.getSizes()); +//// patches.append(parser.getPatchesSourceCode()); +//// alarmTypes.append(parser.getAlarmTypes()); +//// +//// a++; +//// if (a % 100 == 0) { +//// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); +//// FileHelper.outputToFile(buggyTokensFilePath, tokens, true); +//// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); +//// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); +//// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); +//// astEditScripts.setLength(0); +//// tokens.setLength(0); +//// sizes.setLength(0); +//// patches.setLength(0); +//// alarmTypes.setLength(0); +//// System.out.println("Finish of parsing " + a + " files......"); +//// } +//// } +//// } catch (TimeoutException e) { +//// err.println("task timed out"); +//// future.cancel(true /* mayInterruptIfRunning */); +//// } catch (InterruptedException e) { +//// err.println("task interrupted"); +//// } catch (ExecutionException e) { +//// err.println("task aborted"); +//// } finally { +//// executor.shutdownNow(); +//// } +//// } +// +// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); +// FileHelper.outputToFile(buggyTokensFilePath, tokens, true); +// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); +// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); +// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); +// astEditScripts.setLength(0); +// tokens.setLength(0); +// sizes.setLength(0); +// patches.setLength(0); +// alarmTypes.setLength(0); +// +// +//// classifyByAlarmTypes(); +// } +// } private static List getMessageFiles(String gumTreeInput) { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/BaseMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/BaseMessage.java new file mode 100644 index 0000000..dfdbbde --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/BaseMessage.java @@ -0,0 +1,46 @@ +package edu.lu.uni.serval.fixminer.akka; + +/** + * Created by anilkoyuncu on 18/09/2018. + */ +public class BaseMessage { + + private long SECONDS_TO_WAIT; + private int id; + private int threadPoolSize; + + + public BaseMessage(int id,Long timeout) { + this.id = id; + this.SECONDS_TO_WAIT = timeout; +// this.threadPoolSize = threadPoolSize; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + + + public long getSECONDS_TO_WAIT() { + return SECONDS_TO_WAIT; + } + + public void setSECONDS_TO_WAIT(long SECONDS_TO_WAIT) { + this.SECONDS_TO_WAIT = SECONDS_TO_WAIT; + } + + public int getThreadPoolSize() { + return threadPoolSize; + } + + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } + + +} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java similarity index 90% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java index c504528..aaa6159 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.compare; import akka.actor.ActorRef; import akka.actor.ActorSystem; @@ -19,7 +19,7 @@ public class AkkaTreeParser { private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class); - public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, String cursor){ + public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, int cursor, String eDiffTimeout){ final List listOfPairs = getMessages(innerPool,cursor); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" @@ -28,7 +28,7 @@ public class AkkaTreeParser { ActorSystem system = null; ActorRef parsingActor = null; - final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool); + final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,eDiffTimeout); try { log.info("Akka begins..."); system = ActorSystem.create("Compare-EnhancedDiff-System"); @@ -41,7 +41,7 @@ public class AkkaTreeParser { } } - public static List getMessages(JedisPool innerPool, String cursor){ + public static List getMessages(JedisPool innerPool, int cursor){ ScanResult scan; @@ -54,7 +54,7 @@ public class AkkaTreeParser { ScanParams sc = new ScanParams(); //150000000 log.info("Scanning "); - sc.count(Integer.valueOf(cursor)); + sc.count(cursor); sc.match("pair_[0-9]*"); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java similarity index 94% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java index cdfcc45..58337b4 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.akka.compare; import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.model.Action; @@ -71,7 +71,7 @@ public class Compare { if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); -// log.info(matchKey); + log.info("{} tagged to be similar" ,matchKey); jedis.select(1); jedis.set(matchKey, result); @@ -86,7 +86,7 @@ public class Compare { } catch (Exception e) { - log.error(e.toString() + " {}", name); + log.debug("{} not comparable", name); // e.printStackTrace(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/RunnableCompare.java similarity index 90% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/RunnableCompare.java index f51ca09..4ee2ced 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/RunnableCompare.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.akka.compare; import redis.clients.jedis.JedisPool; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeActor.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java similarity index 94% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeActor.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java index 6957275..17f660e 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.compare; import akka.actor.ActorRef; import akka.actor.Props; @@ -60,7 +60,7 @@ public class TreeActor extends UntypedActor { int toIndex = (i + 1) * average + counter; List pairsOfWorkers = pairs.subList(fromIndex, toIndex); - final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool); + final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool,((TreeMessage) message).getSECONDS_TO_WAIT()); mineRouter.tell(workMsg, getSelf()); logger.info("Assign a task to worker #" + (i + 1) + "..."); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeMessage.java similarity index 64% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeMessage.java index b23bfe8..a99752c 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeMessage.java @@ -1,5 +1,6 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.compare; +import edu.lu.uni.serval.fixminer.akka.BaseMessage; import redis.clients.jedis.JedisPool; import java.util.List; @@ -7,25 +8,27 @@ import java.util.List; /** * Created by anilkoyuncu on 12/09/2018. */ -public class TreeMessage { - private int id; +public class TreeMessage extends BaseMessage{ private List name; private JedisPool innerPool; private JedisPool outerPool; - public TreeMessage(int id, List name, JedisPool innerPool, JedisPool outerPool) { - this.id = id; + + public TreeMessage(int id, List name, JedisPool innerPool, JedisPool outerPool,String eDiffTimeout) { + super(id,new Long(eDiffTimeout)); + this.name = name; this.innerPool = innerPool; this.outerPool = outerPool; - } - public int getId() { - return id; } + public TreeMessage(int id, List name, JedisPool innerPool, JedisPool outerPool,Long eDiffTimeout) { + super(id,eDiffTimeout); + + this.name = name; + this.innerPool = innerPool; + this.outerPool = outerPool; - public void setId(int id) { - this.id = id; } public List getName() { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java similarity index 93% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeWorker.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java index 5eb61e1..41849f7 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java @@ -1,11 +1,9 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.compare; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; import edu.lu.uni.serval.config.Configuration; -import edu.lu.uni.serval.fixminer.cluster.Compare; -import edu.lu.uni.serval.fixminer.cluster.RunnableCompare; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.JedisPool; @@ -66,7 +64,7 @@ public class TreeWorker extends UntypedActor { if(message instanceof TreeMessage) { -// if (message instanceof edu.lu.uni.serval.fixminer.cluster.akka.EDiffMessage) { +// if (message instanceof edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage) { TreeMessage msg = (TreeMessage) message; List files = msg.getName(); JedisPool innerPool = msg.getInnerPool(); @@ -92,12 +90,12 @@ public class TreeWorker extends UntypedActor { // }.counter; // - final ExecutorService executor = Executors.newFixedThreadPool(20); + final ExecutorService executor = Executors.newWorkStealingPool(); // // schedule the work final Future future = executor.submit(new RunnableCompare(name, innerPool, outerPool, new Compare())); try { // wait for task to complete - future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); + future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS); Compare compare = new Compare(); compare.coreCompare(name, innerPool, outerPool); counter++; @@ -157,9 +155,8 @@ public class TreeWorker extends UntypedActor { } } - log.info("bitti"); - log.info("Worker #" + id +"finialized parsing " + counter + " files..."); - log.info("Worker #" + id + " finialized the work..."); +// log.info("Worker #" + id +"finialized parsing " + counter + " files..."); + log.info("Worker #" + id + " finalized the work..."); this.getSender().tell("STOP", getSelf()); // String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; // stopServer = String.format(stopServer,Integer.valueOf(innerPort)); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffActor.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java similarity index 88% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffActor.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java index 42be0b0..9a354be 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.ediff; import akka.actor.ActorRef; import akka.actor.Props; @@ -54,15 +54,15 @@ public class EDiffActor extends UntypedActor { int toIndex = (i + 1) * average + counter; List filesOfWorkers = files.subList(fromIndex, toIndex); - final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers); + final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getActionType()); mineRouter.tell(workMsg, getSelf()); logger.info("Assign a task to worker #" + (i + 1) + "..."); } } else if ("STOP".equals(message.toString())) { counter ++; - logger.info(counter + " workers finailized their work..."); + logger.info(counter + " workers finalized their work..."); if (counter >= numberOfWorkers) { - logger.info("All workers finailized their work..."); + logger.info("All workers finalized their work..."); this.getContext().stop(mineRouter); this.getContext().stop(getSelf()); this.getContext().system().shutdown(); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternHunkParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java similarity index 55% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternHunkParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java index eaf7e7c..c2dba4b 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java @@ -1,4 +1,9 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; + +import com.github.gumtreediff.actions.model.Delete; +import com.github.gumtreediff.actions.model.Insert; +import com.github.gumtreediff.actions.model.Move; +import com.github.gumtreediff.actions.model.Update; import java.io.*; import java.util.List; @@ -11,7 +16,7 @@ import java.util.List; * @author kui.liu * */ -public class FixedPatternHunkParser extends FixedPatternParser { +public class EDiffHunkParser extends EDiffParser { public String testingInfo = ""; @@ -25,30 +30,49 @@ public class FixedPatternHunkParser extends FixedPatternParser { public String unfixedViolations = ""; @Override - public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project) { + public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project,String actionType) { List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); if (actionSets.size() != 0) { -// boolean isUpdate = -// actionSets.stream().allMatch(p -> p.getAction() instanceof Update); -// boolean isInsert = -// actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); -// boolean isDelete = -// actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); -// boolean isMove = -// actionSets.stream().allMatch(p -> p.getAction() instanceof Move); + String folder= null; + boolean processActionSet = false; + switch (actionType){ + case "ALL": + folder = "/ALL/"; + processActionSet = true; + break; + case "UPD": + processActionSet = + actionSets.stream().allMatch(p -> p.getAction() instanceof Update); + folder = "/UPD/"; + break; + case "INS": + processActionSet = + actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); + + folder = "/INS/"; + break; + case "DEL": + processActionSet = + actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); + folder = "/DEL/"; + break; + case "MOV": + processActionSet = + actionSets.stream().allMatch(p -> p.getAction() instanceof Move); + folder = "/MOV/"; + break; + default: + processActionSet = false; + System.err.print(actionType + "not known"); + break; + } + + int hunkSet = 0; -// if (isUpdate || isInsert || isDelete || isMove) { + if(processActionSet){ for (HierarchicalActionSet actionSet : actionSets) { - String folder = "/ALL/"; -// if (isUpdate) { -// folder = "/UPD/"; -// } else if (isDelete) { -// folder = "/DEL/"; -// } else if (isInsert) { -// folder = "/INS/"; -// } else if (isMove) { -// folder = "/MOV/"; -// } + + FileOutputStream f = null; @@ -75,7 +99,7 @@ public class FixedPatternHunkParser extends FixedPatternParser { hunkSet++; } -// } + } } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java new file mode 100644 index 0000000..ac5e11d --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java @@ -0,0 +1,45 @@ +package edu.lu.uni.serval.fixminer.akka.ediff; + +import edu.lu.uni.serval.fixminer.akka.BaseMessage; + +import java.util.List; + +public class EDiffMessage extends BaseMessage{ + + + private List msgFiles; + + + + private String actionType; + + + + + + public EDiffMessage(int id, List msgFiles,String eDiffTimeout,String actionType) { + super(id,new Long(eDiffTimeout)); + this.msgFiles = msgFiles; + this.actionType = actionType; + } + public EDiffMessage(int id, List msgFiles,Long eDiffTimeout,String actionType) { + super(id,eDiffTimeout); + this.msgFiles = msgFiles; + this.actionType = actionType; + } + + public String getActionType() { + return actionType; + } + + public void setActionType(String actionType) { + this.actionType = actionType; + } + + + + public List getMsgFiles() { + return msgFiles; + } + +} \ No newline at end of file diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java similarity index 79% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java index ae3255a..8e0e9d7 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import com.github.gumtreediff.actions.model.Action; import edu.lu.uni.serval.gumtree.GumTreeComparer; @@ -14,7 +14,7 @@ import java.util.List; * @author kui.liu * */ -public class FixedPatternParser extends Parser { +public class EDiffParser extends Parser { /* * ResultType: @@ -26,11 +26,7 @@ public class FixedPatternParser extends Parser { */ public int resultType = 0; - protected String violationTypes = ""; - @Override - public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project) { - } /** * Regroup GumTree results without remove the modification of variable names. @@ -65,16 +61,11 @@ public class FixedPatternParser extends Parser { } } + @Override + public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, String actionType) { + + } - -// @Override -// public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { -// -// } - -// public void setUselessViolations(List uselessViolations) { -// this.uselessViolations = uselessViolations; -// } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java similarity index 83% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffWorker.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java index 5c8a77f..00fb5f0 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java @@ -1,11 +1,8 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.ediff; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; -import edu.lu.uni.serval.FixPatternParser.FixedPatternHunkParser; -import edu.lu.uni.serval.FixPatternParser.RunnableParser; -import edu.lu.uni.serval.config.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,14 +65,15 @@ public class EDiffWorker extends UntypedActor { File diffentryFile = msgFile.getDiffEntryFile(); - FixedPatternHunkParser parser = new FixedPatternHunkParser(); + + EDiffHunkParser parser = new EDiffHunkParser(); - final ExecutorService executor = Executors.newSingleThreadExecutor(); + final ExecutorService executor = Executors.newWorkStealingPool(); // schedule the work - final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project)); + final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getActionType())); try { // wait for task to complete - future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); + future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS); nullDiffEntry += parser.nullMatchedDiffEntry; nullMappingGumTreeResults += parser.nullMappingGumTreeResult; @@ -101,7 +99,7 @@ public class EDiffWorker extends UntypedActor { patchesSourceCode.setLength(0); sizes.setLength(0); tokens.setLength(0); - log.info("Worker #" + id +" finialized parsing " + counter + " files..."); + log.info("Worker #" + id +" finalized parsing " + counter + " files..."); testingInfo.setLength(0); } } @@ -128,8 +126,8 @@ public class EDiffWorker extends UntypedActor { testingInfo.setLength(0); } - log.info("Worker #" + id +"finialized parsing " + counter + " files..."); - log.info("Worker #" + id + " finialized the work..."); +// log.info("Worker #" + id +" finalized parsing " + counter + " files..."); + log.info("Worker #" + id + " finalized the work..."); this.getSender().tell("STOP", getSelf()); } else { unhandled(message); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalActionSet.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalActionSet.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java index a0125b6..3a019e0 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalActionSet.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.tree.ITree; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalRegrouper.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java similarity index 99% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalRegrouper.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java index 004c773..5549e2f 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalRegrouper.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.tree.ITree; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/MessageFile.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java similarity index 93% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/MessageFile.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java index e2bea96..b51e597 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/MessageFile.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.ediff; import java.io.File; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java similarity index 92% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java index 2b7367b..21bd9b4 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import java.io.File; @@ -18,7 +18,7 @@ public abstract class Parser implements ParserInterface { protected String originalTree = ""; // Guide of generating patches. protected String actionSets = ""; // Guide of generating patches. - public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile,String project); + public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile,String project,String actionType); @Override diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java similarity index 88% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java index 2a4c517..aac3e0f 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; public interface ParserInterface { diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java similarity index 80% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java index 891af4f..b74726f 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import java.io.File; @@ -9,6 +9,7 @@ public class RunnableParser implements Runnable { private File diffentryFile; private Parser parser; private String project; + private String actionType; public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) { this.prevFile = prevFile; @@ -17,16 +18,17 @@ public class RunnableParser implements Runnable { this.parser = parser; } - public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser,String project) { + public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser,String project,String actionType) { this.prevFile = prevFile; this.revFile = revFile; this.diffentryFile = diffentryFile; this.parser = parser; this.project = project; + this.actionType= actionType; } @Override public void run() { - parser.parseFixPatterns(prevFile, revFile, diffentryFile,project); + parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,actionType); } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffMessage.java deleted file mode 100644 index afea756..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffMessage.java +++ /dev/null @@ -1,24 +0,0 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; - -import java.util.List; - -public class EDiffMessage { - - private int id; - private List msgFiles; - - public EDiffMessage(int id, List msgFiles) { - super(); - this.id = id; - this.msgFiles = msgFiles; - } - - public int getId() { - return id; - } - - public List getMsgFiles() { - return msgFiles; - } - -} \ No newline at end of file diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java index 439c845..ecbdf8d 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java @@ -14,7 +14,7 @@ import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; -import static edu.lu.uni.serval.fixminer.cluster.akka.AkkaTreeParser.akkaCompare; +import static edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser.akkaCompare; /** * Created by anilkoyuncu on 19/03/2018. @@ -24,7 +24,7 @@ public class AkkaTreeLoader { private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class); - public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, String cursor,String chunk) throws Exception { + public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, int cursor,String chunk,String eDiffTimeout) throws Exception { String parameters = String.format("\nportInner %s \nchunkName %s \ndbDir %s \ndumpsName %s",portInner,chunkName,dbDir,dumpsName); @@ -90,7 +90,17 @@ public class AkkaTreeLoader { - akkaCompare(innerPool,outerPool,numOfWorkers,cursor); + akkaCompare(innerPool,outerPool,numOfWorkers,cursor,eDiffTimeout); + + String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; + String stopServer2 = String.format(stopServer,Integer.valueOf(port)); + + cs.runShell(stopServer2,port); + + + String stopServer1 = String.format(stopServer,Integer.valueOf(portInner)); + + cs.runShell(stopServer1,portInner); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java index dfaa3c0..494857d 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java @@ -1,8 +1,8 @@ package edu.lu.uni.serval.fixminer.jobs; import edu.lu.uni.serval.utils.CallShell; -import edu.lu.uni.serval.utils.PoolBuilder; import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -22,7 +22,7 @@ import java.util.List; public class CalculatePairs { private static Logger log = LoggerFactory.getLogger(CalculatePairs.class); - public static void main(String dbDir,String chunkName,String port,String outputPath,String pjName) throws Exception { + public static void main(String dbDir,String chunkName,String port,String outputPath,String pjName,boolean isBigPair,int cursor) throws Exception { String parameters = String.format("\nport %s \nchunkName %s \ndbDir %s",port,chunkName,dbDir); @@ -46,7 +46,7 @@ public class CalculatePairs { ScanParams sc = new ScanParams(); //150000000 - sc.count(1500000000); + sc.count(cursor); sc.match("*"); scan = outer.scan("0", sc); @@ -60,31 +60,59 @@ public class CalculatePairs { byte [] buf = new byte[0]; String line = null; + if(isBigPair) { + bigPair(outputPath, pjName, result, buf, line); + }else{ + smallPair(outputPath, pjName, result, buf, line); + } + +// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; +// String stopServer2 = String.format(stopServer,Integer.valueOf(port)); +//// loadRedis(stopServer2,serverWait); +// cs.runShell(stopServer2,serverWait); + log.info("Done pairs"); + } + + private static void smallPair(String outputPath, String pjName, List result, byte[] buf, String line) { try { -// FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv"); -// DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); -// + FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv"); + DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); + + FileOutputStream fosIndex = new FileOutputStream(outputPath + "/" +pjName+".index"); + DataOutputStream outStreamIndex = new DataOutputStream(new BufferedOutputStream(fosIndex)); + + for (int i = 0; i < result.size(); i++) { + line = String.valueOf(i) +"," + result.get(i)+"\n"; + outStreamIndex.write(line.getBytes()); + + for (int j = i + 1; j < result.size(); j++) { + + + + line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n"; + outStream.write(line.getBytes()); + + } + } + outStream.close(); + outStreamIndex.close(); + + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + }catch (java.nio.BufferOverflowException e) { + log.error(line); + log.error(String.valueOf(buf.length)); + e.printStackTrace(); + } + } + + private static void bigPair(String outputPath, String pjName, List result, byte[] buf, String line) { + try { + FileOutputStream fosIndex = new FileOutputStream(outputPath + "/" +pjName+".index"); DataOutputStream outStreamIndex = new DataOutputStream(new BufferedOutputStream(fosIndex)); -// -// -// -// -// for (int i = 0; i < result.size(); i++) { -// line = String.valueOf(i) +"," + result.get(i)+"\n"; -// outStreamIndex.write(line.getBytes()); -// -// for (int j = i + 1; j < result.size(); j++) { -// -// -// -// line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n"; -// outStream.write(line.getBytes()); -// -// } -// } -// outStream.close(); -// outStreamIndex.close(); int fileCounter = 0; FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +pjName +String.valueOf(fileCounter)+".txt", "rw").getChannel(); int maxSize = 500*1000000; @@ -97,8 +125,6 @@ public class CalculatePairs { for (int j = i + 1; j < result.size(); j++) { - -// line = String.valueOf(i) +"\t" + String.valueOf(j)+"\n"; // + "\t" + result.get(i) + "\t" + result.get(j)+"\n"; line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n"; buf = line.getBytes(); if(wrBuf.remaining() > 500) { @@ -126,12 +152,6 @@ public class CalculatePairs { log.error(String.valueOf(buf.length)); e.printStackTrace(); } - -// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; -// String stopServer2 = String.format(stopServer,Integer.valueOf(port)); -//// loadRedis(stopServer2,serverWait); -// cs.runShell(stopServer2,serverWait); - log.info("Done pairs"); } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java index 0ec8660..febe383 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java @@ -2,9 +2,9 @@ package edu.lu.uni.serval.fixminer.jobs; import akka.actor.ActorRef; import akka.actor.ActorSystem; -import edu.lu.uni.serval.fixminer.cluster.akka.EDiffMessage; -import edu.lu.uni.serval.fixminer.cluster.akka.MessageFile; -import edu.lu.uni.serval.fixminer.cluster.akka.EDiffActor; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage; +import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile; import edu.lu.uni.serval.utils.FileHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,7 +20,7 @@ public class EnhancedASTDiff { private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class); - public static void main(String inputPath, String outputPath,String numOfWorkers,String project) { + public static void main(String inputPath, String outputPath, String numOfWorkers, String project, String eDiffTimeout, String actionType) { String parameters = String.format("\nInput path %s \nOutput path %s",inputPath,outputPath); @@ -45,18 +45,19 @@ public class EnhancedASTDiff { String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ALL"); +// a +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/"+actionType); int a = 0; ActorSystem system = null; ActorRef parsingActor = null; - final EDiffMessage msg = new EDiffMessage(0, msgFiles); + final EDiffMessage msg = new EDiffMessage(0, msgFiles,eDiffTimeout,actionType); try { log.info("Akka begins..."); system = ActorSystem.create("Mining-FixPattern-System"); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java index bf2befb..79bc574 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java @@ -44,7 +44,7 @@ public class ImportPairs2DB { cmd = String.format(cmd, pj.getPath(), portInt); log.info(cmd); - cs.runShell(cmd); + cs.runShell(cmd,portInner); String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; String stopServer2 = String.format(stopServer,portInt); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java index 723babf..3feed61 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java @@ -6,12 +6,8 @@ import com.github.gumtreediff.matchers.Matcher; import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.utils.CallShell; -import edu.lu.uni.serval.utils.EDiff; -import edu.lu.uni.serval.utils.EDiffHelper; -import edu.lu.uni.serval.utils.PoolBuilder; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; -import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; +import edu.lu.uni.serval.utils.*; import org.javatuples.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,7 +32,7 @@ public class MultiThreadTreeLoaderCluster { private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoaderCluster.class); - public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type) throws Exception { + public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type,int cursor) throws Exception { CallShell cs = new CallShell(); String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; @@ -74,7 +70,7 @@ public class MultiThreadTreeLoaderCluster { try (Jedis jedis = jedisPool.getResource()) { // do operations with jedis resource ScanParams sc = new ScanParams(); - sc.count(150000000); + sc.count(cursor); sc.match("pair_[0-9]*"); log.info("Scanning"); @@ -84,13 +80,14 @@ public class MultiThreadTreeLoaderCluster { if (size == 0) { String comd = String.format(cmd3,f.getPath(),portInner); // loadRedis(comd); - cs.runShell(comd); + log.info("Importing {} pairs for cluster {}",size,f.getName()); + cs.runShell(comd,portInner); scan = jedis.scan("0", sc); size = scan.getResult().size(); } - log.info("Scanned " + String.valueOf(size)); + log.info("Scanned {} for cluster {}",String.valueOf(size),f.getName()); String clusterName = f.getName().replaceAll("[^0-9]+", ""); @@ -247,7 +244,7 @@ public class MultiThreadTreeLoaderCluster { // log.info("Completed " + resultKey); }catch (Exception e){ - log.warn(e.toString() + " {}",(name)); + log.debug(e.toString() + " {}",(name)); } @@ -329,7 +326,7 @@ public class MultiThreadTreeLoaderCluster { e.printStackTrace(); } - log.info("Done pairs"); + log.info("Done pairs of cluster {}",cluster); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java index 73562b8..880189d 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java @@ -2,11 +2,11 @@ package edu.lu.uni.serval.fixminer.jobs; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import edu.lu.uni.serval.utils.CallShell; import edu.lu.uni.serval.utils.EDiffHelper; -import edu.lu.uni.serval.utils.PoolBuilder; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.similarity.JaroWinklerDistance; import org.slf4j.Logger; @@ -33,7 +33,7 @@ public class MultiThreadTreeLoaderCluster3 { private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoaderCluster3.class); - public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type) throws Exception { + public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type,int cursor) throws Exception { CallShell cs = new CallShell(); String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; @@ -69,7 +69,7 @@ public class MultiThreadTreeLoaderCluster3 { try (Jedis jedis = jedisPool.getResource()) { // do operations with jedis resource ScanParams sc = new ScanParams(); - sc.count(150000000); + sc.count(cursor); sc.match("pair_[0-9]*"); log.info("Scanning"); @@ -78,7 +78,7 @@ public class MultiThreadTreeLoaderCluster3 { if (size == 0) { String comd = String.format(cmd3, f.getPath(), portInner); - cs.runShell(comd); + cs.runShell(comd,portInner); scan = jedis.scan("0", sc); @@ -149,7 +149,7 @@ public class MultiThreadTreeLoaderCluster3 { ITree parent = null; ITree children =null; TreeContext tc = new TreeContext(); - tree = EDiffHelper.getASTTree(actionSet, parent, children,tc); + tree = EDiffHelper.getTokenTree(actionSet, parent, children,tc); // tree.setParent(null); tc.validate(); @@ -526,7 +526,7 @@ public class MultiThreadTreeLoaderCluster3 { .collect(Collectors.toList()); FileHelper.createDirectory(outputPath + "/pairs-2l"+type); - + log.info("Calculating pairs"); for (File pj : pjs) { File[] files = pj.listFiles(); List fileList = Arrays.asList(files); @@ -536,6 +536,7 @@ public class MultiThreadTreeLoaderCluster3 { } File[] clusterFiles = cluster.listFiles(); List clusterFilesL = Arrays.asList(clusterFiles); + readMessageFilesCluster(clusterFilesL, outputPath, inputPath, pj.getName(), cluster.getName(),type); @@ -559,7 +560,7 @@ public class MultiThreadTreeLoaderCluster3 { treesFileNames.add(target.toString()); } - log.info("Calculating pairs"); + // treesFileNames = treesFileNames.subList(0,100); String filename = "cluster" + cluster + "_" + subCluster; @@ -609,7 +610,7 @@ public class MultiThreadTreeLoaderCluster3 { e.printStackTrace(); } - log.info("Done pairs"); + log.info("Done pairs of {}",filename); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java index 0dd8b6e..1386f94 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java @@ -1,7 +1,7 @@ package edu.lu.uni.serval.fixminer.jobs; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import edu.lu.uni.serval.utils.Checker; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java index 9e5541d..4acb113 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java @@ -1,9 +1,9 @@ package edu.lu.uni.serval.fixminer.jobs; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import edu.lu.uni.serval.utils.CallShell; import edu.lu.uni.serval.utils.EDiffHelper; import edu.lu.uni.serval.utils.PoolBuilder; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -112,10 +112,10 @@ public class StoreEDiffInCache { } catch (FileNotFoundException e) { - log.error("File not found"); + log.error("File not found {}" , path); e.printStackTrace(); } catch (IOException e) { - log.error("Error initializing stream"); + log.error("Error initializing stream {}" , path); e.printStackTrace(); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java index 99a32fe..3c0818f 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java @@ -22,7 +22,7 @@ public class TreeLoaderClusterL1 { private static Logger log = LoggerFactory.getLogger(TreeLoaderClusterL1.class); - public static void main(String portInner,String port,String inputPath,String level1DB,String level1Path,String innerTypePrefix) throws Exception { + public static void main(String portInner,String port,String inputPath,String level1DB,String level1Path,String innerTypePrefix,int cursor) throws Exception { @@ -59,7 +59,7 @@ public class TreeLoaderClusterL1 { inner.select(1); ScanParams sc = new ScanParams(); //150000000 - sc.count(150000000); + sc.count(cursor); sc.match("match_[0-9]*"); ScanResult scan; scan = inner.scan("0", sc); diff --git a/src/main/java/edu/lu/uni/serval/utils/CallShell.java b/src/main/java/edu/lu/uni/serval/utils/CallShell.java index 9e4b56d..85c76ab 100644 --- a/src/main/java/edu/lu/uni/serval/utils/CallShell.java +++ b/src/main/java/edu/lu/uni/serval/utils/CallShell.java @@ -4,6 +4,9 @@ package edu.lu.uni.serval.utils; * Created by anilkoyuncu on 17/04/2018. */ +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -11,6 +14,7 @@ import java.util.concurrent.TimeUnit; public class CallShell { + private static Logger log = LoggerFactory.getLogger(CallShell.class); @@ -28,6 +32,8 @@ public class CallShell { public static void runShell(String command, String port) throws Exception { + log.info(command); + Process process = Runtime.getRuntime().exec(command); BufferedReader reader = new BufferedReader(new InputStreamReader( process.getInputStream())); diff --git a/src/main/java/edu/lu/uni/serval/utils/EDiff.java b/src/main/java/edu/lu/uni/serval/utils/EDiff.java index a2a1bfc..01fd931 100644 --- a/src/main/java/edu/lu/uni/serval/utils/EDiff.java +++ b/src/main/java/edu/lu/uni/serval/utils/EDiff.java @@ -3,7 +3,7 @@ package edu.lu.uni.serval.utils; import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java index ae7d3f0..74cf247 100644 --- a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java +++ b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java @@ -2,7 +2,7 @@ package edu.lu.uni.serval.utils; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +38,47 @@ public class EDiffHelper { return Base64.getEncoder().encodeToString(baos.toByteArray()); } + + public static ITree getTokenTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){ + + int newType = 0; + + String astNodeType = actionSet.getAstNodeType(); + + String label = actionSet.getAction().toString(); + List keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType); + + if(keysByValue.size() != 1){ + log.error("More than 1"); + } + newType = keysByValue.get(0); + if(actionSet.getParent() == null){ + //root + + parent = tc.createTree(newType, label, null); + tc.setRoot(parent); + }else{ + + children = tc.createTree(newType, label, null); + children.setParentAndUpdateChildren(parent); + } + List subActions = actionSet.getSubActions(); + if (subActions.size() != 0){ + for (HierarchicalActionSet subAction : subActions) { + + if(actionSet.getParent() == null){ + children = parent; + } + getTokenTree(subAction,children,null,tc); + + } + + + } + return parent; + } + + public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children, TreeContext tc){ int newType = 0; From 711f8776c6a50234c766be41fdd3c9b956f741e9 Mon Sep 17 00:00:00 2001 From: fixminer Date: Sat, 22 Sep 2018 15:26:49 +0200 Subject: [PATCH 3/4] parallelism selection --- .../lu/uni/serval/config/Configuration.java | 7 - .../edu/lu/uni/serval/fixminer/Launcher.java | 15 +- .../fixminer/MultiThreadTreeLoader.java | 600 ------------------ .../fixminer/TestHunkParserSingleFile.java | 239 ------- .../fixminer/akka/compare/AkkaTreeParser.java | 57 +- .../serval/fixminer/akka/compare/Compare.java | 10 +- .../fixminer/akka/compare/TreeActor.java | 6 +- .../fixminer/akka/compare/TreeWorker.java | 158 +---- .../fixminer/akka/ediff/EDiffActor.java | 2 +- .../fixminer/akka/ediff/EDiffHunkParser.java | 57 +- .../fixminer/akka/ediff/EDiffWorker.java | 61 +- .../serval/fixminer/jobs/AkkaTreeLoader.java | 22 +- .../serval/fixminer/jobs/EnhancedASTDiff.java | 95 ++- .../serval/fixminer/jobs/ImportPairs2DB.java | 7 +- .../jobs/MultiThreadTreeLoaderCluster.java | 7 +- .../jobs/MultiThreadTreeLoaderCluster3.java | 2 +- .../fixminer/jobs/PatternExtractor.java | 76 +-- .../fixminer/jobs/StoreEDiffInCache.java | 30 +- .../java/edu/lu/uni/serval/utils/EDiff.java | 98 --- .../edu/lu/uni/serval/utils/EDiffHelper.java | 80 +++ 20 files changed, 288 insertions(+), 1341 deletions(-) delete mode 100644 src/main/java/edu/lu/uni/serval/config/Configuration.java delete mode 100644 src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java delete mode 100644 src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java delete mode 100644 src/main/java/edu/lu/uni/serval/utils/EDiff.java diff --git a/src/main/java/edu/lu/uni/serval/config/Configuration.java b/src/main/java/edu/lu/uni/serval/config/Configuration.java deleted file mode 100644 index 4421c72..0000000 --- a/src/main/java/edu/lu/uni/serval/config/Configuration.java +++ /dev/null @@ -1,7 +0,0 @@ -package edu.lu.uni.serval.config; - -public class Configuration { - - public static final long SECONDS_TO_WAIT = 900L; - -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java index 2dec952..a4cbbd0 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java @@ -39,23 +39,24 @@ public class Launcher { String chunk = appProps.getProperty("chunk","1.txt"); String eDiffTimeout = appProps.getProperty("eDiffTimeout","900"); String isBig = appProps.getProperty("isBigPair","true"); + String parallelism = appProps.getProperty("parallelism","FORKJOIN"); boolean isBigPair = Boolean.parseBoolean(isBig); String parameters = String.format("\nportInner %s " + "\nnumOfWorkers %s " + "\njobType %s \nport %s " + "\npythonPath %s \ndatasetPath %s" + - "\npjName %s \nactionType %s \nthreshold %s \ncursor %s \neDiffTimeout %s \nisBigPair %s" - , portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,eDiffTimeout,isBig); + "\npjName %s \nactionType %s \nthreshold %s \ncursor %s \neDiffTimeout %s \nisBigPair %s \nparallelism %s" + , portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,eDiffTimeout,isBig,parallelism); log.info(parameters); - mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk,eDiffTimeout,isBigPair); + mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk,eDiffTimeout,isBigPair,parallelism); } - public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk, String eDiffTimeout, boolean isBigPair){ + public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk, String eDiffTimeout, boolean isBigPair, String parallelism){ String dbDir; @@ -74,17 +75,17 @@ public class Launcher { try { switch (jobType) { case "ENHANCEDASTDIFF": - EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName, eDiffTimeout,actionType); + EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName, eDiffTimeout,actionType,parallelism); break; case "CACHE": StoreEDiffInCache.main(gumOutput, portDumps, dbDir, actionType+dumpsName,actionType); break; case "SI": CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType,isBigPair,iCursor); - ImportPairs2DB.main(pairsPath+actionType, portInner, dbDir,datasetPath); + ImportPairs2DB.main(pairsPath+actionType, portInner, dbDir,datasetPath,chunk); break; case "SIMI": - AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,iCursor,chunk,eDiffTimeout); + AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,iCursor,chunk,eDiffTimeout,parallelism); break; case "LEVEL1": diff --git a/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java deleted file mode 100644 index 8734393..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java +++ /dev/null @@ -1,600 +0,0 @@ -package edu.lu.uni.serval.fixminer; - -import com.github.gumtreediff.actions.ActionGenerator; -import com.github.gumtreediff.actions.model.Action; -import com.github.gumtreediff.matchers.Matcher; -import com.github.gumtreediff.matchers.Matchers; -import com.github.gumtreediff.tree.ITree; -import com.github.gumtreediff.tree.Tree; -import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; -import edu.lu.uni.serval.utils.ASTNodeMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; - -import java.io.*; -import java.nio.file.Files; -import java.time.Duration; -import java.util.*; -import java.util.concurrent.Executors; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Created by anilkoyuncu on 19/03/2018. - */ -public class MultiThreadTreeLoader { - - private static class StreamGobbler implements Runnable { - private InputStream inputStream; - private Consumer consumer; - - public StreamGobbler(InputStream inputStream, Consumer consumer) { - this.inputStream = inputStream; - this.consumer = consumer; - } - - @Override - public void run() { - new BufferedReader(new InputStreamReader(inputStream)).lines() - .forEach(consumer); - } - } - - private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoader.class); - - - public static void loadRedis(String cmd,String serverWait){ - Process process; - - try { -// String comd = String.format(cmd, f.getAbsoluteFile()); - process = Runtime.getRuntime() - - .exec(cmd); - - - StreamGobbler streamGobbler = - new StreamGobbler(process.getInputStream(), System.out::println); - Executors.newSingleThreadExecutor().submit(streamGobbler); -// int exitCode = process.waitFor(); -// assert exitCode == 0; - Thread.sleep(Integer.valueOf(serverWait)); - - } catch (IOException e) { - e.printStackTrace(); - } - catch (InterruptedException e) { - e.printStackTrace(); - } - log.info("Load done"); - } - - public static void main(String[] args) { - - - String inputPath; - String outputPath; - String port; - String portInner; - String pairsCSVPath; - String importScript; - String pairsCompletedPath; - String serverWait; - String option; - String dbDir; - String chunkName; - if (args.length > 0) { - inputPath = args[0]; - port = args[1]; - portInner = args[2]; - serverWait = args[3]; - option = args[4]; - chunkName = args[5]; - dbDir = args[6]; -// pairsCSVPath = args[3]; -// importScript = args[4]; -// pairsCompletedPath = args[3]; - } else { - inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; - outputPath = "/Users/anilkoyuncu/bugStudy/dataset/"; - port = "6379"; - portInner = "6380"; - serverWait = "10000"; - option = "COMP"; - pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test"; - importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh"; - pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed"; - chunkName ="chunk5.rdb"; - dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; - } - - if (option.equals("CALC")) { - calculatePairs(inputPath, port); - log.info("Calculate pairs done"); - }else { - comparePairs(port, inputPath, portInner, serverWait,chunkName,dbDir); - } - - - } - - - public static void comparePairs(String port,String inputPath, String innerPort,String serverWait,String chunkName, String dbDir){ -// String cmd; -// cmd = "bash " + importScript +" %s"; - - - - List dir; - List path; - - String orgDbname; - - - File files = new File(dbDir); - File[] listFiles = files.listFiles(); - - - Stream stream = Arrays.stream(listFiles); - List folders = stream - .filter(x -> x.getName().equals(chunkName)) - .collect(Collectors.toList()); - for (File folder : folders) { - - - - - String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; - cmd = String.format(cmd, dbDir,folder.getName(),Integer.valueOf(innerPort)); - loadRedis(cmd,serverWait); - - - JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000); - ScanResult scan; - try (Jedis inner = pool.getResource()) { - while (inner.ping()== "PONG"){ - log.info("wait"); - } - - ScanParams sc = new ScanParams(); - sc.count(150000000); - sc.match("pair_*"); - - scan = inner.scan("0", sc); - int size = scan.getResult().size(); - log.info("Scanning " + String.valueOf(size)); - } - scan.getResult().parallelStream() - .forEach(m -> coreCompare(m, inputPath, innerPort)); - - - String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; - stopServer = String.format(stopServer,Integer.valueOf(innerPort)); - loadRedis(stopServer,serverWait); - - } - - - - - } - - private static void coreCompare(String name , String inputPath, String innerPort) { - JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000); - Map resultMap; - try (Jedis jedis = pool.getResource()) { - resultMap = jedis.hgetAll(name); - } - String[] split = name.split("_"); - - - String i = split[1]; - String j = split[2]; - String firstValue = resultMap.get("0"); - String secondValue = resultMap.get("1"); - - String[] firstValueSplit = firstValue.split("GumTreeOutput2"); - String[] secondValueSplit = secondValue.split("GumTreeOutput2"); - - if (firstValueSplit.length == 1) { - firstValue = inputPath + firstValueSplit[0]; - } else { - firstValue = inputPath + firstValueSplit[1]; - } - - if (secondValueSplit.length == 1) { - secondValue = inputPath + secondValueSplit[0]; - } else { - secondValue = inputPath + secondValueSplit[1]; - } - - try { - ITree oldTree = getSimpliedTree(firstValue); - - ITree newTree = getSimpliedTree(secondValue); - - Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); - m.match(); - - - ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); - ag.generate(); - List actions = ag.getActions(); - - double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree); - String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1); - double diceSimilarity1 = m.diceSimilarity(oldTree, newTree); - String diceSimilarity = String.format("%1.2f", diceSimilarity1); - double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree); - String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); - - String editDistance = String.valueOf(actions.size()); - - String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; - - - if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) - || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { - String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); - - try (Jedis jedis = pool.getResource()) { - jedis.select(1); - jedis.set(matchKey, result); - } - } - - - try (Jedis jedis = pool.getResource()) { - jedis.del("pair_"+ (String.valueOf(i)) + "_" + String.valueOf(j)); - } - - - - - }catch (Exception e){ - log.warn(e.toString() + " {}",(name)); - - - } - - - - - - } - - - - public static void calculatePairs(String inputPath,String port) { - File folder = new File(inputPath); - File[] listOfFiles = folder.listFiles(); - Stream stream = Arrays.stream(listOfFiles); - List pjs = stream - .filter(x -> !x.getName().startsWith(".")) - .collect(Collectors.toList()); - - List fileToCompare = new ArrayList<>(); - for (File pj : pjs) { - File[] files = pj.listFiles(new FilenameFilter() { - public boolean accept(File dir, String name) { - return name.startsWith("ActionSetDumps"); - } - }); - Collections.addAll(fileToCompare, files[0].listFiles()); - - } - System.out.println("a"); -// compareAll(fileToCompare); - readMessageFiles(fileToCompare,port); - } - -// public static void processMessages(String inputPath, String outputPath) { -// File folder = new File(outputPath + "pairs_splitted/"); -// File[] listOfFiles = folder.listFiles(); -// Stream stream = Arrays.stream(listOfFiles); -// List pjs = stream -// .filter(x -> !x.getName().startsWith(".")) -// .collect(Collectors.toList()); -// FileHelper.createDirectory(outputPath + "comparison_splitted/"); -// pjs.parallelStream() -// .forEach(m -> coreLoop(m, outputPath,inputPath)); -// } - - - public static ITree getSimpliedTree(String fn) { - HierarchicalActionSet actionSet = null; - try { - FileInputStream fi = new FileInputStream(new File(fn)); - ObjectInputStream oi = new ObjectInputStream(fi); - actionSet = (HierarchicalActionSet) oi.readObject(); - oi.close(); - fi.close(); - - - } catch (FileNotFoundException e) { - log.error("File not found"); - e.printStackTrace(); - } catch (IOException e) { - log.error("Error initializing stream"); - e.printStackTrace(); - } catch (ClassNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - ITree parent = null; - ITree children =null; - ITree tree = getASTTree(actionSet, parent, children); - tree.setParent(null); - - return tree; - - } - - public static List getKeysByValue(Map map, E value) { - return map.entrySet() - .stream() - .filter(entry -> Objects.equals(entry.getValue(), value)) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - } - - public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children){ - - int newType = 0; - - String astNodeType = actionSet.getAstNodeType(); - List keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType); - - if(keysByValue.size() != 1){ - log.error("Birden cok astnodemapmapping"); - } - newType = keysByValue.get(0); - if(actionSet.getParent() == null){ - //root - - parent = new Tree(newType,""); - }else{ - children = new Tree(newType,""); - parent.addChild(children); - } - List subActions = actionSet.getSubActions(); - if (subActions.size() != 0){ - for (HierarchicalActionSet subAction : subActions) { - - if(actionSet.getParent() == null){ - children = parent; - } - getASTTree(subAction,children,null); - - } - - - } - return parent; - } - -// public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children){ -// -// int newType = 0; -// -// Action action = actionSet.getAction(); -// if (action instanceof Update){ -// newType = 101; -// }else if(action instanceof Insert){ -// newType =100; -// }else if(action instanceof Move){ -// newType = 102; -// }else if(action instanceof Delete){ -// newType=103; -// }else{ -// new Exception("unknow action"); -// } -// if(actionSet.getParent() == null){ -// //root -// -// parent = new Tree(newType,""); -// }else{ -// children = new Tree(newType,""); -// parent.addChild(children); -// } -// List subActions = actionSet.getSubActions(); -// if (subActions.size() != 0){ -// for (HierarchicalActionSet subAction : subActions) { -// -// if(actionSet.getParent() == null){ -// children = parent; -// } -// getActionTree(subAction,children,null); -// -// } -// -// -// } -// return parent; -// } - - - - private static void coreLoop(File mes, String outputPath,String inputPath) { - try { - - log.info("Starting in coreLoop"); - - BufferedReader br = null; - String sCurrentLine = null; - BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "comparison_splitted/" + "output_" + mes.getName())); - - br = new BufferedReader( - new FileReader(mes)); - while ((sCurrentLine = br.readLine()) != null) { - String currentLine = sCurrentLine; - String[] split = currentLine.split("\t"); - String i = split[0]; - String j = split[1]; - String firstValue = split[2]; - String secondValue = split[3]; - - firstValue = inputPath + firstValue.split("GumTreeOutput2")[1]; - secondValue = inputPath + secondValue.split("GumTreeOutput2")[1]; - - ITree oldTree = getSimpliedTree(firstValue); - - ITree newTree = getSimpliedTree(secondValue); - - Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); - m.match(); - - ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); - ag.generate(); - List actions = ag.getActions(); - writer.write(String.valueOf(i)); - writer.write("\t"); - writer.write(String.valueOf(j)); - writer.write("\t"); - - writer.write(String.format("%1.2f", m.chawatheSimilarity(oldTree, newTree))); - writer.write("\t"); - writer.write(String.format("%1.2f", m.diceSimilarity(oldTree, newTree))); - writer.write("\t"); - writer.write(String.format("%1.2f", m.jaccardSimilarity(oldTree, newTree))); - writer.write("\t"); - writer.write(String.valueOf(actions.size())); - writer.write("\t"); - writer.write(firstValue); - writer.write("\t"); - writer.write(secondValue); - writer.write("\n"); - - - } - writer.close(); - } catch (FileNotFoundException e) { - log.error("File not found"); - e.printStackTrace(); - } catch (IOException e) { - log.error("Error initializing stream"); - e.printStackTrace(); - - } - log.info("Completed output_" + mes.getName()); - } - - private static void readMessageFiles(List folders,String port) { - - List treesFileNames = new ArrayList<>(); - - - for (File target : folders) { - - treesFileNames.add(target.toString()); - } -// FileHelper.createDirectory(outputPath + "pairs/"); - log.info("Calculating pairs"); -// treesFileNames = treesFileNames.subList(0,100); - byte [] buf = new byte[0]; - String line = null; - - -// FileChannel rwChannel = new RandomAccessFile(outputPath + "pairs/" +"textfile.txt", "rw").getChannel(); -// ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE); - int fileCounter = 0; - - JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); - try (Jedis jedis = jedisPool.getResource()) { - List dir = null; - List path = null; - for (int i = 0; i < treesFileNames.size(); i++) { - for (int j = i + 1; j < treesFileNames.size(); j++) { - - - // do operations with jedis resource - - String key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j); -// String value = treesFileNames.get(i).split("GumTreeOutput2")[1] +","+treesFileNames.get(j).split("GumTreeOutput2")[1]; -// jedis.set(key,value); - - jedis.hset(key,"0",treesFileNames.get(i).split("GumTreeOutput2")[1]); - jedis.hset(key,"1",treesFileNames.get(j).split("GumTreeOutput2")[1]); - //10000000 - if(Integer.compare(jedis.dbSize().intValue(),10000000) == 0){ - dir = jedis.configGet("dir"); - path = jedis.configGet("dbfilename"); - File dbPath = new File(dir.get(1)+"/"+path.get(1)); - File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb"); - try { - jedis.save(); - while (jedis.ping()== "PONG"){ - log.info("wait"); - } - - - - Files.copy(dbPath.toPath(),savePath.toPath()); - } catch (IOException e) { - - e.printStackTrace(); - } - fileCounter++; - jedis.flushDB(); - - } - - - } - } - jedis.save(); - fileCounter++; - File dbPath = new File(dir.get(1)+"/"+path.get(1)); - File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb"); - try { - - while (jedis.ping()== "PONG"){ - log.info("wait"); - } - - - - Files.copy(dbPath.toPath(),savePath.toPath()); - } catch (IOException e) { - - e.printStackTrace(); - } - jedis.flushDB(); - } - - - - - log.info("Done pairs"); - } - - static final JedisPoolConfig poolConfig = buildPoolConfig(); - - - private static JedisPoolConfig buildPoolConfig() { - final JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(128); - poolConfig.setMaxIdle(128); - poolConfig.setMinIdle(16); - poolConfig.setTestOnBorrow(true); - poolConfig.setTestOnReturn(true); - poolConfig.setTestWhileIdle(true); - poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); - poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); - poolConfig.setNumTestsPerEvictionRun(3); - poolConfig.setBlockWhenExhausted(true); - - return poolConfig; - } - - - -// return msgFiles; -} - - - - diff --git a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java deleted file mode 100644 index f116deb..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java +++ /dev/null @@ -1,239 +0,0 @@ -package edu.lu.uni.serval.fixminer; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor; -import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage; -import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile; -import edu.lu.uni.serval.utils.FileHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class TestHunkParserSingleFile { - - private static Logger log = LoggerFactory.getLogger(TestHunkParserSingleFile.class); -// public static void main(String[] args) { -// // input data -// -//// String rootPath = "/Users/anilkoyuncu/bugStudy"; -// String inputPath; -// String outputPath; -// if(args.length > 0){ -// inputPath = args[1]; -// outputPath = args[0]; -// }else{ -//// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeInputBug4"; -// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/allDataset"; -//// outputPath = "/Users/anilkoyuncu/bugStudy/code/python/GumTreeOutput2/"; -// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputSingle"; -// } -// -// //5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.txt_1_CAMEL -// -// File folder = new File(inputPath); -// File[] listOfFiles = folder.listFiles(); -// Stream stream = Arrays.stream(listOfFiles); -// List folders = stream -// .filter(x -> !x.getName().startsWith(".")) -// -// .collect(Collectors.toList()); -// -//// List targetList = new ArrayList(); -//// for (File f:folders){ -//// for(File f1 :f.listFiles()){ -//// if (!f1.getName().startsWith(".")){ -//// targetList.add(f1); -//// } -//// } -//// } -// -//// List pjList = Arrays.asList("DATAJPA","ZXing","PDE","SWS","SWT", "SWF", "COLLECTIONS", "JDT"); -// List files = new ArrayList(); -//// files.add("5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.java"); -// files.add("d6c5e5_9f96d9_hbase-server#src#main#java#org#apache#hadoop#hbase#master#RegionStates.java"); -// for(String f : files){ -// String pjName = "HBASE"; -//// for (File target : folders) { -//// String pjName = target.getName(); -//// if (!pjList.contains(pjName)){ -//// continue; -//// } -// -//// final List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" -// MessageFile messageFile = getMessageFile(inputPath + "/" + pjName +"/", f); -// -// List msgFiles = new ArrayList<>(); -// msgFiles.add(messageFile); -// String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; -// final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list"; -// final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list"; -// final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list"; -// final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv"; -// final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list"; -// -// -// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ActionSetDumps"); -// FileHelper.deleteDirectory(editScriptsFilePath); -// FileHelper.deleteDirectory(patchesSourceCodeFilePath); -// FileHelper.deleteDirectory(buggyTokensFilePath); -// FileHelper.deleteDirectory(editScriptSizesFilePath); -// FileHelper.deleteDirectory(alarmTypesFilePath); -// -// StringBuilder astEditScripts = new StringBuilder(); -// StringBuilder tokens = new StringBuilder(); -// StringBuilder sizes = new StringBuilder(); -// StringBuilder patches = new StringBuilder(); -// StringBuilder alarmTypes = new StringBuilder(); -// -//// int a = 0; -// -// ActorSystem system = null; -// ActorRef parsingActor = null; -// final EDiffMessage msg = new EDiffMessage(0, msgFiles); -// try { -// log.info("Akka begins..."); -// system = ActorSystem.create("Mining-FixPattern-System"); -// parsingActor = system.actorOf(EDiffActor.props(1, "dataset"), "mine-fix-pattern-actor"); -// parsingActor.tell(msg, ActorRef.noSender()); -// } catch (Exception e) { -// system.shutdown(); -// e.printStackTrace(); -// } -// -//// int counter = 0; -//// for (MessageFile msgFile : msgFiles) { -//// EDiffHunkParser parser = new EDiffHunkParser(); -//// -//// final ExecutorService executor = Executors.newSingleThreadExecutor(); -//// // schedule the work -//// final Future future = executor.submit(new RunnableParser(msgFile.getPrevFile(), -//// msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser)); -//// try { -//// // where we wait for task to complete -//// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); -//// String editScripts = parser.getAstEditScripts(); -//// if (!editScripts.equals("")) { -//// astEditScripts.append(editScripts); -//// tokens.append(parser.getTokensOfSourceCode()); -//// sizes.append(parser.getSizes()); -//// patches.append(parser.getPatchesSourceCode()); -//// alarmTypes.append(parser.getAlarmTypes()); -//// -//// a++; -//// if (a % 100 == 0) { -//// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); -//// FileHelper.outputToFile(buggyTokensFilePath, tokens, true); -//// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); -//// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); -//// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); -//// astEditScripts.setLength(0); -//// tokens.setLength(0); -//// sizes.setLength(0); -//// patches.setLength(0); -//// alarmTypes.setLength(0); -//// System.out.println("Finish of parsing " + a + " files......"); -//// } -//// } -//// } catch (TimeoutException e) { -//// err.println("task timed out"); -//// future.cancel(true /* mayInterruptIfRunning */); -//// } catch (InterruptedException e) { -//// err.println("task interrupted"); -//// } catch (ExecutionException e) { -//// err.println("task aborted"); -//// } finally { -//// executor.shutdownNow(); -//// } -//// } -// -// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); -// FileHelper.outputToFile(buggyTokensFilePath, tokens, true); -// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); -// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); -// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); -// astEditScripts.setLength(0); -// tokens.setLength(0); -// sizes.setLength(0); -// patches.setLength(0); -// alarmTypes.setLength(0); -// -// -//// classifyByAlarmTypes(); -// } -// } - - - private static List getMessageFiles(String gumTreeInput) { - String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile - File revFilesPath = new File(inputPath + "revFiles/"); - File[] revFiles = revFilesPath.listFiles(); // project folders - List msgFiles = new ArrayList<>(); - - // gumTreeInput = /Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/AMQP/ - // fileName = 01534a_df5570_spring-rabbit#src#test#java#org#springframework#amqp#rabbit#listener#LocallyTransactedTests.java - if (revFiles.length >= 0) { - for (File revFile : revFiles) { -// if (revFile.getName().endsWith(".java")) { - String fileName = revFile.getName(); - File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file - fileName = fileName.replace(".java", ".txt"); - File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file - File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file - MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); - msgFile.setPositionFile(positionFile); - msgFiles.add(msgFile); -// } - } - - return msgFiles; - } - else{ - return null; - } - } - - private static MessageFile getMessageFile(String gumTreeInput, String fileName) { -// String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile -// File revFilesPath = new File(inputPath + "revFiles/"); -// File[] revFiles = revFilesPath.listFiles(); // project folders -// List msgFiles = new ArrayList<>(); - - // gumTreeInput = /Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/AMQP/ - // fileName = 01534a_df5570_spring-rabbit#src#test#java#org#springframework#amqp#rabbit#listener#LocallyTransactedTests.java -// if (revFiles.length >= 0) { -// for (File revFile : revFiles) { -// if (revFile.getName().endsWith(".java")) { -// String fileName = revFile.getName(); - File revFile = new File(gumTreeInput + "revFiles/"+fileName); - File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file - fileName = fileName.replace(".java", ".txt"); - File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file -// File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file - MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); - return msgFile; -// msgFile.setPositionFile(positionFile); -// msgFiles.add(msgFile); -// } -// } -// -// return msgFiles; -// } -// else{ -// return null; -// } - } - - - - - - - -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java index aaa6159..15cbc87 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java @@ -19,26 +19,57 @@ public class AkkaTreeParser { private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class); - public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, int cursor, String eDiffTimeout){ + public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, int cursor, String eDiffTimeout, String parallelism){ final List listOfPairs = getMessages(innerPool,cursor); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" + switch (parallelism){ + case "AKKA": + ActorSystem system = null; + ActorRef parsingActor = null; + final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,eDiffTimeout); + try { + log.info("Akka begins..."); + system = ActorSystem.create("Compare-EnhancedDiff-System"); + parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor"); + parsingActor.tell(msg, ActorRef.noSender()); + } catch (Exception e) { + system.shutdown(); + e.printStackTrace(); + } + break; + case "FORKJOIN": + int counter = new Object() { + int counter = 0; - ActorSystem system = null; - ActorRef parsingActor = null; - final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,eDiffTimeout); - try { - log.info("Akka begins..."); - system = ActorSystem.create("Compare-EnhancedDiff-System"); - - parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor"); - parsingActor.tell(msg, ActorRef.noSender()); - } catch (Exception e) { - system.shutdown(); - e.printStackTrace(); + { + listOfPairs.stream(). + parallel(). + peek(x -> counter++). + forEach(m -> + { + Compare compare = new Compare(); + compare.coreCompare(m, innerPool, outerPool); + if (counter % 10 == 0) { + log.info("Finalized parsing " + counter + " files... remaing " + (listOfPairs.size() - counter)); + } + } + ); + } + }.counter; + log.info("Finished parsing {} files",counter); + break; + default: + log.error("Unknown parallelism {}", parallelism); + break; } + + + + + } public static List getMessages(JedisPool innerPool, int cursor){ diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java index 58337b4..ef93559 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java @@ -5,7 +5,7 @@ import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.matchers.Matcher; import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; -import edu.lu.uni.serval.utils.EDiff; +import edu.lu.uni.serval.utils.EDiffHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -32,7 +32,6 @@ public class Compare { try { jedis = innerPool.getResource(); -// resultMap = jedis.hgetAll(name); String[] split = name.split("_"); @@ -41,12 +40,9 @@ public class Compare { String j = split[2]; -// String firstValue = resultMap.get("0"); -// String secondValue = resultMap.get("1"); + oldTree = EDiffHelper.getSimpliedTree(i,outerPool); - oldTree = EDiff.getSimpliedTree(i,outerPool); - - newTree = EDiff.getSimpliedTree(j,outerPool); + newTree = EDiffHelper.getSimpliedTree(j,outerPool); Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); m.match(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java index 17f660e..24b2045 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java @@ -62,13 +62,13 @@ public class TreeActor extends UntypedActor { List pairsOfWorkers = pairs.subList(fromIndex, toIndex); final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool,((TreeMessage) message).getSECONDS_TO_WAIT()); mineRouter.tell(workMsg, getSelf()); - logger.info("Assign a task to worker #" + (i + 1) + "..."); + logger.info("Assign {} task to worker #" + (i + 1) + "...",pairsOfWorkers.size()); } } else if ("STOP".equals(message.toString())) { counter ++; - logger.info(counter + " workers finailized their work..."); + logger.info(counter + " workers finalized their work..."); if (counter >= numberOfWorkers) { - logger.info("All workers finailized their work..."); + logger.info("All workers finalized their work..."); this.getContext().stop(mineRouter); this.getContext().stop(getSelf()); this.getContext().system().shutdown(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java index 41849f7..ae79022 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java @@ -3,7 +3,6 @@ package edu.lu.uni.serval.fixminer.akka.compare; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; -import edu.lu.uni.serval.config.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.JedisPool; @@ -15,32 +14,7 @@ public class TreeWorker extends UntypedActor { private static Logger log = LoggerFactory.getLogger(TreeWorker.class); -// private JedisPool innerPool; -// private JedisPool outerPool; -// -// public TreeWorker(String innerPort,String outerPort) { -//// this.innerPool = innerPool; -//// this.outerPool = outerPool; -// this.outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(outerPort),20000000); -// this.innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(innerPort),20000000); -// -// -// } -// -// public static Props props(final String innerPort,final String outerPort) { -// return Props.create(new Creator() { -// -// private static final long serialVersionUID = -7615153844097275009L; -// -// @Override -// public TreeWorker create() throws Exception { -// return new TreeWorker(innerPort,outerPort); -// } -// -// }); -// } - - public TreeWorker() { + public TreeWorker() { } @@ -63,160 +37,46 @@ public class TreeWorker extends UntypedActor { public void onReceive(Object message) throws Exception { if(message instanceof TreeMessage) { - -// if (message instanceof edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage) { TreeMessage msg = (TreeMessage) message; List files = msg.getName(); JedisPool innerPool = msg.getInnerPool(); JedisPool outerPool = msg.getOuterPool(); int id = msg.getId(); -// int counter = new Object() { - int counter = 0; + int counter = 0; - // for (String name : files) { -// files.stream(). -// parallel(). -// peek(x -> counter++). -// forEach(m -> -// { -// Compare compare = new Compare(); -// compare.coreCompare(m, innerPool, outerPool); -// } -// ); -// } -// }.counter; -// - final ExecutorService executor = Executors.newWorkStealingPool(); + + Compare compare = new Compare(); + final ExecutorService executor = Executors.newSingleThreadExecutor(); // // schedule the work - final Future future = executor.submit(new RunnableCompare(name, innerPool, outerPool, new Compare())); + final Future future = executor.submit(new RunnableCompare(name, innerPool, outerPool, compare)); try { // wait for task to complete future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS); - Compare compare = new Compare(); - compare.coreCompare(name, innerPool, outerPool); counter++; -// nullDiffEntry += parser.nullMatchedDiffEntry; -// nullMappingGumTreeResults += parser.nullMappingGumTreeResult; -// pureDeletion += parser.pureDeletions; -// largeHunk += parser.largeHunk; -// nullSourceCode += parser.nullSourceCode; -// testInfos += parser.testInfos; -// testingInfo.append(parser.testingInfo); -// builder.append(parser.unfixedViolations); -// -// String editScript = parser.getAstEditScripts(); -// if ("".equals(editScript)) { -//// if (parser.resultType == 1) { -//// nullGumTreeResults += countAlarms(positionFile, ""); -//// } else if (parser.resultType == 2) { -//// noSourceCodeChanges += countAlarms(positionFile, ""); -//// } else if (parser.resultType == 3) { -//// noStatementChanges += countAlarms(positionFile, ""); -//// } -// } else { -// editScripts.append(editScript); -// patchesSourceCode.append(parser.getPatchesSourceCode()); -// sizes.append(parser.getSizes()); -// tokens.append(parser.getTokensOfSourceCode()); -// -// counter ++; -// if (counter % 100 == 0) { -// FileHelper.outputToFile(editScriptsFilePath + "edistScripts_" + id + ".list", editScripts, true); -// FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true); -// FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true); -// FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true); -// editScripts.setLength(0); -// patchesSourceCode.setLength(0); -// sizes.setLength(0); -// tokens.setLength(0); -// log.info("Worker #" + id +" finialized parsing " + counter + " files..."); -// FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); -// testingInfo.setLength(0); -// } -// } + if (counter % 100 == 0) { + log.info("Worker #" + id +" finalized parsing " + counter + " pairs... remaing "+ (files.size() - counter)); + } } catch (TimeoutException e) { future.cancel(true); -////// timeouts += countAlarms(positionFile, "#Timeout:"); -// System.err.println("#Timeout: " + name); } catch (InterruptedException e) { -////// timeouts += countAlarms(positionFile, "#TimeInterrupted:"); -//// System.err.println("#TimeInterrupted: " + revFile.getName()); e.printStackTrace(); } catch (ExecutionException e) { -////// timeouts += countAlarms(positionFile, "#TimeAborted:"); -//// System.err.println("#TimeAborted: " + revFile.getName()); e.printStackTrace(); } finally { executor.shutdownNow(); } } -// log.info("Worker #" + id +"finialized parsing " + counter + " files..."); log.info("Worker #" + id + " finalized the work..."); this.getSender().tell("STOP", getSelf()); -// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; -// stopServer = String.format(stopServer,Integer.valueOf(innerPort)); -// loadRedis(stopServer,serverWait); -// -// if (sizes.length() > 0) { -// FileHelper.outputToFile(editScriptsFilePath + "editScripts_" + id + ".list", editScripts, true); -// FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true); -// FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true); -// FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true); -// editScripts.setLength(0); -// patchesSourceCode.setLength(0); -// sizes.setLength(0); -// tokens.setLength(0); -// -// FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); -// testingInfo.setLength(0); -// } -// String statistic = "\nNullGumTreeResults: " + nullGumTreeResults + "\nNoSourceCodeChanges: " + noSourceCodeChanges + -// "\nNoStatementChanges: " + noStatementChanges + "\nNullDiffEntry: " + nullDiffEntry + "\nNullMatchedGumTreeResults: " + nullMappingGumTreeResults + -// "\nPureDeletion: " + pureDeletion + "\nLargeHunk: " + largeHunk + "\nNullSourceCode: " + nullSourceCode + -// "\nTestingInfo: " + testInfos + "\nTimeout: " + timeouts; -// FileHelper.outputToFile("OUTPUT/statistic_" + id + ".list", statistic, false); -// FileHelper.outputToFile("OUTPUT/UnfixedV_" + id + ".list", builder, false); -// -// log.info("Worker #" + id +"finialized parsing " + counter + " files..."); -// log.info("Worker #" + id + " finialized the work..."); -// this.getSender().tell("STOP", getSelf()); -// } else { }else{ unhandled(message); } } -// static final JedisPoolConfig poolConfig = buildPoolConfig(); -// -// -// private static JedisPoolConfig buildPoolConfig() { -// final JedisPoolConfig poolConfig = new JedisPoolConfig(); -// poolConfig.setMaxTotal(128); -// poolConfig.setMaxIdle(128); -// poolConfig.setMinIdle(16); -// poolConfig.setTestOnBorrow(true); -// poolConfig.setTestOnReturn(true); -// poolConfig.setTestWhileIdle(true); -// poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); -// poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); -// poolConfig.setNumTestsPerEvictionRun(3); -// poolConfig.setBlockWhenExhausted(true); -// -// return poolConfig; -// } - - - - - - - - - } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java index 9a354be..5d16315 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java @@ -56,7 +56,7 @@ public class EDiffActor extends UntypedActor { List filesOfWorkers = files.subList(fromIndex, toIndex); final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getActionType()); mineRouter.tell(workMsg, getSelf()); - logger.info("Assign a task to worker #" + (i + 1) + "..."); + logger.info("Assign {} task to worker #" + (i + 1) ,filesOfWorkers.size()); } } else if ("STOP".equals(message.toString())) { counter ++; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java index c2dba4b..c2b6309 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java @@ -18,53 +18,45 @@ import java.util.List; */ public class EDiffHunkParser extends EDiffParser { - public String testingInfo = ""; - - public int nullMappingGumTreeResult = 0; - public int pureDeletions = 0; - public int largeHunk = 0; - public int nullSourceCode = 0; - public int nullMatchedDiffEntry = 0; - public int testInfos = 0; - public String unfixedViolations = ""; - @Override public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project,String actionType) { List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); if (actionSets.size() != 0) { String folder= null; boolean processActionSet = false; - switch (actionType){ - case "ALL": +// switch (actionType){ +// case "ALL": + if(actionType.equals("ALL")){ folder = "/ALL/"; processActionSet = true; - break; - case "UPD": - processActionSet = - actionSets.stream().allMatch(p -> p.getAction() instanceof Update); + }else if(actionType.equals("UPD") || actionType.equals("INS") || actionType.equals("DEL") || actionType.equals("MOV")|| actionType.equals("MIX")){ + boolean isUPD = actionSets.stream().allMatch(p -> p.getAction() instanceof Update); + boolean isINS = actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); + boolean isDEL = actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); + boolean isMOV = actionSets.stream().allMatch(p -> p.getAction() instanceof Move); + if(isUPD){ folder = "/UPD/"; - break; - case "INS": - processActionSet = - actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); - + processActionSet = true; + }else if(isINS){ folder = "/INS/"; - break; - case "DEL": - processActionSet = - actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); + processActionSet = true; + }else if(isDEL){ folder = "/DEL/"; - break; - case "MOV": - processActionSet = - actionSets.stream().allMatch(p -> p.getAction() instanceof Move); + processActionSet = true; + }else if(isMOV){ folder = "/MOV/"; - break; - default: + processActionSet = true; + }else{ + folder = "/MIX/"; + processActionSet = true; + } + }else{ + + processActionSet = false; System.err.print(actionType + "not known"); - break; + } @@ -78,7 +70,6 @@ public class EDiffHunkParser extends EDiffParser { FileOutputStream f = null; try { -// String pj = diffentryFile.getParent().split("Defects4J")[1]; String datasetName = project; String[] split1 = diffentryFile.getParent().split(datasetName); String root = split1[0]; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java index 00fb5f0..77a30b0 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java @@ -38,27 +38,10 @@ public class EDiffWorker extends UntypedActor { if (message instanceof EDiffMessage) { EDiffMessage msg = (EDiffMessage) message; List files = msg.getMsgFiles(); - StringBuilder editScripts = new StringBuilder(); - StringBuilder patchesSourceCode = new StringBuilder(); - StringBuilder sizes = new StringBuilder(); - StringBuilder tokens = new StringBuilder(); - StringBuilder testingInfo = new StringBuilder(); int id = msg.getId(); int counter = 0; - int nullGumTreeResults = 0; - int noSourceCodeChanges = 0; - int noStatementChanges = 0; - int nullDiffEntry = 0; - int nullMappingGumTreeResults = 0; - int pureDeletion = 0; - int largeHunk = 0; - int nullSourceCode = 0; - int testInfos = 0; - int timeouts = 0; - StringBuilder builder = new StringBuilder(); - for (MessageFile msgFile : files) { File revFile = msgFile.getRevFile(); File prevFile = msgFile.getPrevFile(); @@ -67,41 +50,17 @@ public class EDiffWorker extends UntypedActor { EDiffHunkParser parser = new EDiffHunkParser(); - - final ExecutorService executor = Executors.newWorkStealingPool(); + + final ExecutorService executor = Executors.newSingleThreadExecutor(); // schedule the work final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getActionType())); try { // wait for task to complete future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS); - nullDiffEntry += parser.nullMatchedDiffEntry; - nullMappingGumTreeResults += parser.nullMappingGumTreeResult; - pureDeletion += parser.pureDeletions; - largeHunk += parser.largeHunk; - nullSourceCode += parser.nullSourceCode; - testInfos += parser.testInfos; - testingInfo.append(parser.testingInfo); - builder.append(parser.unfixedViolations); - - String editScript = parser.getAstEditScripts(); - if ("".equals(editScript)) { - - } else { - editScripts.append(editScript); - patchesSourceCode.append(parser.getPatchesSourceCode()); - sizes.append(parser.getSizes()); - tokens.append(parser.getTokensOfSourceCode()); - - counter ++; - if (counter % 100 == 0) { - editScripts.setLength(0); - patchesSourceCode.setLength(0); - sizes.setLength(0); - tokens.setLength(0); - log.info("Worker #" + id +" finalized parsing " + counter + " files..."); - testingInfo.setLength(0); - } + counter ++; + if (counter % 10 == 0) { + log.info("Worker #" + id +" finalized parsing " + counter + " files... remaing "+ (files.size() - counter)); } } catch (TimeoutException e) { future.cancel(true); @@ -116,17 +75,7 @@ public class EDiffWorker extends UntypedActor { executor.shutdownNow(); } } - - if (sizes.length() > 0) { - editScripts.setLength(0); - patchesSourceCode.setLength(0); - sizes.setLength(0); - tokens.setLength(0); - - testingInfo.setLength(0); - } -// log.info("Worker #" + id +" finalized parsing " + counter + " files..."); log.info("Worker #" + id + " finalized the work..."); this.getSender().tell("STOP", getSelf()); } else { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java index ecbdf8d..e154673 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java @@ -24,7 +24,7 @@ public class AkkaTreeLoader { private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class); - public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, int cursor,String chunk,String eDiffTimeout) throws Exception { + public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, int cursor, String chunk, String eDiffTimeout, String parallelism) throws Exception { String parameters = String.format("\nportInner %s \nchunkName %s \ndbDir %s \ndumpsName %s",portInner,chunkName,dbDir,dumpsName); @@ -90,17 +90,17 @@ public class AkkaTreeLoader { - akkaCompare(innerPool,outerPool,numOfWorkers,cursor,eDiffTimeout); + akkaCompare(innerPool,outerPool,numOfWorkers,cursor,eDiffTimeout,parallelism); - String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; - String stopServer2 = String.format(stopServer,Integer.valueOf(port)); - - cs.runShell(stopServer2,port); - - - String stopServer1 = String.format(stopServer,Integer.valueOf(portInner)); - - cs.runShell(stopServer1,portInner); +// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; +// String stopServer2 = String.format(stopServer,Integer.valueOf(port)); +// +// cs.runShell(stopServer2,port); +// +// +// String stopServer1 = String.format(stopServer,Integer.valueOf(portInner)); +// +// cs.runShell(stopServer1,portInner); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java index febe383..afe421b 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java @@ -3,6 +3,7 @@ package edu.lu.uni.serval.fixminer.jobs; import akka.actor.ActorRef; import akka.actor.ActorSystem; import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser; import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage; import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile; import edu.lu.uni.serval.utils.FileHelper; @@ -20,7 +21,7 @@ public class EnhancedASTDiff { private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class); - public static void main(String inputPath, String outputPath, String numOfWorkers, String project, String eDiffTimeout, String actionType) { + public static void main(String inputPath, String outputPath, String numOfWorkers, String project, String eDiffTimeout, String actionType, String parallelism) { String parameters = String.format("\nInput path %s \nOutput path %s",inputPath,outputPath); @@ -34,43 +35,81 @@ public class EnhancedASTDiff { .collect(Collectors.toList()); + List allMessageFiles = new ArrayList<>(); for (File target : folders) { - String pjName = target.getName(); + String pjName = target.getName(); - final List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" - System.out.println(msgFiles.size()); - if(msgFiles.size() == 0) - continue; + List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" + allMessageFiles.addAll(msgFiles); +// System.out.println(msgFiles.size()); + if (msgFiles.size() == 0) + continue; - String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; + String GUM_TREE_OUTPUT = outputPath + "/" + pjName + "/"; // a -// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD"); -// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); -// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); -// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/"+actionType); - - int a = 0; - - ActorSystem system = null; - ActorRef parsingActor = null; - final EDiffMessage msg = new EDiffMessage(0, msgFiles,eDiffTimeout,actionType); - try { - log.info("Akka begins..."); - system = ActorSystem.create("Mining-FixPattern-System"); - - parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor"); - parsingActor.tell(msg, ActorRef.noSender()); - } catch (Exception e) { - system.shutdown(); - e.printStackTrace(); + if (actionType.equals("ALL")) { + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/" + actionType); + } else { + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MIX"); } + } + + switch (parallelism){ + case "AKKA": + ActorSystem system = null; + ActorRef parsingActor = null; + final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,actionType); + try { + log.info("Akka begins..."); + log.info("{} files to process ...", allMessageFiles.size()); + system = ActorSystem.create("Mining-FixPattern-System"); + + parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor"); + parsingActor.tell(msg, ActorRef.noSender()); + } catch (Exception e) { + system.shutdown(); + e.printStackTrace(); + } + break; + case "FORKJOIN": + int counter = new Object() { + int counter = 0; + + { + allMessageFiles.stream(). + parallel(). + peek(x -> counter++). + forEach(m -> + { + EDiffHunkParser parser = new EDiffHunkParser(); + parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,actionType); + if (counter % 10 == 0) { + log.info("Finalized parsing " + counter + " files... remaing " + (allMessageFiles.size() - counter)); + } + } + ); + } + }.counter; + log.info("Finished parsing {} files",counter); + break; + default: + log.error("Unknown parallelism {}", parallelism); + break; + } + + + + } - } + private static List getMessageFiles(String gumTreeInput) { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java index 79bc574..e3a04c7 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java @@ -17,18 +17,21 @@ import java.util.stream.Stream; public class ImportPairs2DB { private static Logger log = LoggerFactory.getLogger(ImportPairs2DB.class); - public static void main(String csvInputPath,String portInner,String dbDir,String datasetPath) throws Exception { + public static void main(String csvInputPath, String portInner, String dbDir, String datasetPath, String chunk) throws Exception { String parameters = String.format("\nInput path %s \nportInner %s \ndbDir %s",csvInputPath,portInner,dbDir); log.info(parameters); + String[] splits = chunk.split("\\."); + String chunkType = splits[splits.length-1]; + log.info("Chunk type {}",chunkType); File folder = new File(csvInputPath); File[] subFolders = folder.listFiles(); Stream stream = Arrays.stream(subFolders); List pjs = stream - .filter(x -> x.getName().endsWith(".txt")) + .filter(x -> x.getName().endsWith(chunkType)) .collect(Collectors.toList()); Integer portInt = Integer.valueOf(portInner); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java index 3feed61..9b993ad 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java @@ -7,7 +7,10 @@ import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; -import edu.lu.uni.serval.utils.*; +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.javatuples.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -145,7 +148,7 @@ public class MultiThreadTreeLoaderCluster { ITree parent = null; ITree children =null; TreeContext tc = new TreeContext(); - tree = EDiff.getActionTree(actionSet, parent, children,tc); + tree = EDiffHelper.getActionTree(actionSet, parent, children,tc); tree.setParent(null); tc.validate(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java index 880189d..e542cc5 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java @@ -413,7 +413,7 @@ public class MultiThreadTreeLoaderCluster3 { } // if (oldTokens.size() == 0 ) {// && (oldTree.getType() != 41 && oldTree.getType() != 21 && oldTree.getType() !=17 && oldTree.getType()!=60 && oldTree.getType() != 46)){ -// log.info("dur bakalim nereye!???"); +// log.info("!???"); // } return oldTokens; } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java index 1386f94..7c7d3bc 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java @@ -2,19 +2,18 @@ package edu.lu.uni.serval.fixminer.jobs; import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import edu.lu.uni.serval.utils.Checker; +import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; -import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; -import java.util.Base64; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -65,36 +64,9 @@ public class PatternExtractor { public static void loadDB(String inputPath,String portInner,String dbDir,String chunkName,String operation,List fixes) throws Exception { -// String inputPath; -// String portInner; -// String serverWait; -// String dbDir; -// String chunkName; -// String numOfWorkers; -// if (args.length > 0) { -// inputPath = args[0]; -// portInner = args[1]; -// serverWait = args[2]; -// chunkName = args[3]; -// numOfWorkers = args[4]; -// dbDir = args[5]; -// } else { -// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; -// portInner = "6399"; -// serverWait = "10000"; -// chunkName ="dumps.rdb"; -// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; -// numOfWorkers = "1"; -// } -// String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s \noperation %s", inputPath, portInner, serverWait, chunkName, dbDir, operation); -// log.info(parameters); -// CallShell cs = new CallShell(); -// String cmd = "bash " + dbDir + "/" + "startServer.sh" + " %s %s %s"; -// cmd = String.format(cmd, dbDir, chunkName, Integer.valueOf(portInner)); -//// loadRedis(cmd,serverWait); -// cs.runShell(cmd, serverWait); - JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(6399),20000000); + + JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(6399),20000000); @@ -134,11 +106,11 @@ public class PatternExtractor { s = inner.get(fn); if (s != null) { - HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s); BufferedWriter writer = new BufferedWriter(new FileWriter(savePath+saveFN)); - writer.write(toString(actionSet)); + writer.write(EDiffHelper.toString(actionSet)); writer.close(); @@ -207,7 +179,7 @@ public class PatternExtractor { String saveFN = pattern.getName() + "_" + project + "_" + operation + "_" + joinFN; try{ String content = new String(Files.readAllBytes(Paths.get(savePath + saveFN))); - HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(content); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(content); int astType = actionSet.getNode().getType(); if (Checker.isStatement(astType) || astType == 23 //FieldDeclaration @@ -264,41 +236,7 @@ public class PatternExtractor { - static final JedisPoolConfig poolConfig = buildPoolConfig(); - private static String toString( Serializable o ) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream( baos ); - oos.writeObject( o ); - oos.close(); - return Base64.getEncoder().encodeToString(baos.toByteArray()); - } - - /** Read the object from Base64 string. */ - private static Object fromString( String s ) throws IOException , - ClassNotFoundException { - byte[] data = Base64.getDecoder().decode(s); - ObjectInputStream ois = new ObjectInputStream( - new ByteArrayInputStream(data)); - Object o = ois.readObject(); - ois.close(); - return o; - } - private static JedisPoolConfig buildPoolConfig() { - final JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(128); - poolConfig.setMaxIdle(128); - poolConfig.setMinIdle(16); - poolConfig.setTestOnBorrow(false); - poolConfig.setTestOnReturn(false); - poolConfig.setTestWhileIdle(true); - poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); - poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); -// poolConfig.setNumTestsPerEvictionRun(3); - poolConfig.setBlockWhenExhausted(true); - - return poolConfig; - } public static List readIdList(String fileName) { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java index 4acb113..2fbb676 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java @@ -48,24 +48,24 @@ public class StoreEDiffInCache { File[] files = pj.listFiles(); Stream fileStream = Arrays.stream(files); List fs; - if (operation.equals("ALLOP")){ - fs= fileStream - .filter(x -> x.getName().startsWith("UPD") || - x.getName().startsWith("INS") || - x.getName().startsWith("DEL") || - x.getName().startsWith("MOV")) - .collect(Collectors.toList()); - File[] files1 = fs.get(0).listFiles(); - File[] files2 = fs.get(1).listFiles(); - File[] files3 = fs.get(2).listFiles(); - File[] files4 = fs.get(3).listFiles(); - dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new); - }else{ +// if (operation.equals("ALLOP")){ +// fs= fileStream +// .filter(x -> x.getName().startsWith("UPD") || +// x.getName().startsWith("INS") || +// x.getName().startsWith("DEL") || +// x.getName().startsWith("MOV")) +// .collect(Collectors.toList()); +// File[] files1 = fs.get(0).listFiles(); +// File[] files2 = fs.get(1).listFiles(); +// File[] files3 = fs.get(2).listFiles(); +// File[] files4 = fs.get(3).listFiles(); +// dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new); +// }else{ fs = fileStream .filter(x -> x.getName().startsWith(operation)) .collect(Collectors.toList()); dumps = fs.get(0).listFiles(); - } +// } for (File f : dumps) { @@ -115,7 +115,7 @@ public class StoreEDiffInCache { log.error("File not found {}" , path); e.printStackTrace(); } catch (IOException e) { - log.error("Error initializing stream {}" , path); + log.error("Error initializing stream {}", path); e.printStackTrace(); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block diff --git a/src/main/java/edu/lu/uni/serval/utils/EDiff.java b/src/main/java/edu/lu/uni/serval/utils/EDiff.java deleted file mode 100644 index 01fd931..0000000 --- a/src/main/java/edu/lu/uni/serval/utils/EDiff.java +++ /dev/null @@ -1,98 +0,0 @@ -package edu.lu.uni.serval.utils; - -import com.github.gumtreediff.actions.model.*; -import com.github.gumtreediff.tree.ITree; -import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; - -import java.io.IOException; -import java.util.List; - -/** - * Created by anilkoyuncu on 17/09/2018. - */ -public class EDiff { - private static Logger log = LoggerFactory.getLogger(EDiff.class); - public static ITree getSimpliedTree(String fn, JedisPool outerPool) { - - ITree tree = null; - Jedis inner = null; - try { - inner = outerPool.getResource(); - while (!inner.ping().equals("PONG")){ - log.info("wait"); - } - inner.select(1); - String dist2load = inner.get(fn); - inner.select(0); - String s = inner.get(dist2load); - HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s); - - ITree parent = null; - ITree children =null; - TreeContext tc = new TreeContext(); - tree = EDiffHelper.getASTTree(actionSet, parent, children,tc); - tree.setParent(null); - tc.validate(); - - } catch (IOException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - }finally { - if (inner != null) { - inner.close(); - } - } - return tree; - - } - public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){ - - int newType = 0; - - Action action = actionSet.getAction(); - if (action instanceof Update){ - newType = 101; - }else if(action instanceof Insert){ - newType =100; - }else if(action instanceof Move){ - newType = 102; - }else if(action instanceof Delete){ - newType=103; - }else{ - new Exception("unknow action"); - } - if(actionSet.getParent() == null){ - //root - - parent = tc.createTree(newType, "", null); - tc.setRoot(parent); - -// parent = new Tree(newType,""); - }else{ - children = tc.createTree(newType, "", null); - children.setParentAndUpdateChildren(parent); -// children = new Tree(newType,""); -// parent.addChild(children); - } - List subActions = actionSet.getSubActions(); - if (subActions.size() != 0){ - for (HierarchicalActionSet subAction : subActions) { - - if(actionSet.getParent() == null){ - children = parent; - } - getActionTree(subAction,children,null,tc); - - } - - - } - return parent; - } -} diff --git a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java index 74cf247..10fddfe 100644 --- a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java +++ b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java @@ -1,10 +1,13 @@ package edu.lu.uni.serval.utils; +import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; import java.io.*; import java.util.Base64; @@ -126,4 +129,81 @@ public class EDiffHelper { .map(Map.Entry::getKey) .collect(Collectors.toList()); } + + public static ITree getSimpliedTree(String fn, JedisPool outerPool) { + + ITree tree = null; + Jedis inner = null; + try { + inner = outerPool.getResource(); + while (!inner.ping().equals("PONG")){ + log.info("wait"); + } + inner.select(1); + String dist2load = inner.get(fn); + inner.select(0); + String s = inner.get(dist2load); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s); + + ITree parent = null; + ITree children =null; + TreeContext tc = new TreeContext(); + tree = EDiffHelper.getASTTree(actionSet, parent, children,tc); + tree.setParent(null); + tc.validate(); + + } catch (IOException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + }finally { + if (inner != null) { + inner.close(); + } + } + return tree; + + } + public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){ + + int newType = 0; + + Action action = actionSet.getAction(); + if (action instanceof Update){ + newType = 101; + }else if(action instanceof Insert){ + newType =100; + }else if(action instanceof Move){ + newType = 102; + }else if(action instanceof Delete){ + newType=103; + }else{ + new Exception("unknow action"); + } + if(actionSet.getParent() == null){ + //root + + parent = tc.createTree(newType, "", null); + tc.setRoot(parent); + + }else{ + children = tc.createTree(newType, "", null); + children.setParentAndUpdateChildren(parent); + + } + List subActions = actionSet.getSubActions(); + if (subActions.size() != 0){ + for (HierarchicalActionSet subAction : subActions) { + + if(actionSet.getParent() == null){ + children = parent; + } + getActionTree(subAction,children,null,tc); + + } + + + } + return parent; + } } From ea934421505f9349b15545e8be5c2e8198eb9b92 Mon Sep 17 00:00:00 2001 From: mimic Date: Sun, 3 Mar 2019 11:42:44 +0100 Subject: [PATCH 4/4] new version --- .idea/.name | 1 + .idea/compiler.xml | 27 + .idea/encodings.xml | 12 + ...__ch_qos_logback_logback_classic_1_1_2.xml | 13 + ...__ch_qos_logback_logback_classic_1_1_7.xml | 13 + ...__ch_qos_logback_logback_classic_1_2_3.xml | 13 + ...ven__ch_qos_logback_logback_core_1_1_2.xml | 13 + ...ven__ch_qos_logback_logback_core_1_1_7.xml | 13 + ...ven__ch_qos_logback_logback_core_1_2_3.xml | 13 + ...thub_mpkorstanje_simmetrics_core_3_0_3.xml | 13 + ..._com_google_code_findbugs_jsr305_1_3_9.xml | 13 + .../Maven__com_google_code_gson_gson_2_3.xml | 13 + .../Maven__com_google_guava_guava_11_0_2.xml | 13 + .../Maven__com_google_guava_guava_18_0.xml | 13 + .../Maven__com_nanohttpd_nanohttpd_2_1_1.xml | 13 + ...om_nanohttpd_nanohttpd_webserver_2_1_1.xml | 13 + .../Maven__com_rabbitmq_amqp_client_4_0_0.xml | 13 + .../Maven__com_sparkjava_spark_core_2_6_0.xml | 13 + ...m_typesafe_akka_akka_actor_2_11_2_4_11.xml | 13 + .../Maven__com_typesafe_config_1_3_0.xml | 13 + ...aven__commons_codec_commons_codec_1_10.xml | 13 + ...Maven__commons_codec_commons_codec_1_9.xml | 13 + .idea/libraries/Maven__dom4j_dom4j_1_6_1.xml | 13 + ..._javax_servlet_javax_servlet_api_3_1_0.xml | 13 + .idea/libraries/Maven__junit_junit_3_8_1.xml | 13 + .idea/libraries/Maven__junit_junit_4_11.xml | 13 + .idea/libraries/Maven__junit_junit_4_12.xml | 13 + .idea/libraries/Maven__log4j_log4j_1_2_14.xml | 13 + .../Maven__net_sf_trove4j_trove4j_3_0_3.xml | 13 + ...__net_sourceforge_jexcelapi_jxl_2_6_12.xml | 13 + ...__org_apache_commons_commons_lang3_3_1.xml | 13 + ...__org_apache_commons_commons_lang3_3_7.xml | 13 + ...org_apache_commons_commons_pool2_2_4_2.xml | 13 + ...n__org_apache_commons_commons_text_1_3.xml | 13 + .../Maven__org_apache_poi_poi_3_12.xml | 13 + .../Maven__org_apache_poi_poi_ooxml_3_12.xml | 13 + ..._org_apache_poi_poi_ooxml_schemas_3_12.xml | 13 + ...en__org_apache_xmlbeans_xmlbeans_2_6_0.xml | 13 + ...e_core_resources_3_10_0_v20150423_0755.xml | 13 + ...ore_contenttype_3_4_200_v20140207_1251.xml | 13 + ...eclipse_core_jobs_3_6_0_v20140424_0053.xml | 13 + ...pse_core_runtime_3_10_0_v20140318_2214.xml | 13 + ...pse_equinox_app_1_3_200_v20130910_1609.xml | 13 + ..._equinox_common_3_6_200_v20130402_1505.xml | 13 + ...nox_preferences_3_5_200_v20140224_1527.xml | 13 + ...quinox_registry_3_5_400_v20140428_1507.xml | 13 + ...pse_jetty_jetty_client_9_4_4_v20170414.xml | 13 + ...lipse_jetty_jetty_http_9_4_4_v20170414.xml | 13 + ...eclipse_jetty_jetty_io_9_4_4_v20170414.xml | 13 + ...e_jetty_jetty_security_9_4_4_v20170414.xml | 13 + ...pse_jetty_jetty_server_9_4_4_v20170414.xml | 13 + ...se_jetty_jetty_servlet_9_4_4_v20170414.xml | 13 + ...lipse_jetty_jetty_util_9_4_4_v20170414.xml | 13 + ...pse_jetty_jetty_webapp_9_4_4_v20170414.xml | 13 + ...clipse_jetty_jetty_xml_9_4_4_v20170414.xml | 13 + ...ebsocket_websocket_api_9_4_4_v20170414.xml | 13 + ...ocket_websocket_client_9_4_4_v20170414.xml | 13 + ...ocket_websocket_common_9_4_4_v20170414.xml | 13 + ...ocket_websocket_server_9_4_4_v20170414.xml | 13 + ...cket_websocket_servlet_9_4_4_v20170414.xml | 13 + ...org_eclipse_osgi_3_10_0_v20140606_1445.xml | 13 + ...eclipse_jdt_core_3_12_2_v20161117_1814.xml | 13 + .../Maven__org_hamcrest_hamcrest_core_1_3.xml | 13 + ...ven__org_javassist_javassist_3_16_1_GA.xml | 13 + .../Maven__org_javatuples_javatuples_1_2.xml | 13 + ..._org_reflections_reflections_0_9_9_RC1.xml | 13 + ...Maven__org_rendersnake_rendersnake_1_8.xml | 13 + ..._modules_scala_java8_compat_2_11_0_7_0.xml | 13 + ...n__org_scala_lang_scala_library_2_11_8.xml | 13 + .../Maven__org_slf4j_slf4j_api_1_7_13.xml | 13 + .../Maven__org_slf4j_slf4j_api_1_7_21.xml | 13 + .../Maven__org_slf4j_slf4j_api_1_7_25.xml | 13 + .../Maven__org_slf4j_slf4j_api_1_7_7.xml | 13 + .../Maven__redis_clients_jedis_2_8_1.xml | 13 + .../libraries/Maven__stax_stax_api_1_0_1.xml | 13 + .../Maven__xml_apis_xml_apis_1_0_b2.xml | 13 + .idea/misc.xml | 15 + .idea/modules.xml | 14 + .idea/vcs.xml | 8 + .idea/workspace.xml | 1218 +++++++++++++++++ FixPatternMiner.iml | 21 +- difffile | 210 +++ pom.xml | 12 +- .../edu/lu/uni/serval/fixminer/Launcher.java | 12 +- .../fixminer/akka/compare/AkkaTreeParser.java | 3 +- .../serval/fixminer/akka/compare/Compare.java | 9 +- .../fixminer/akka/compare/CompareTrees.java | 197 +++ .../fixminer/akka/ediff/EDiffHunkParser.java | 78 +- .../akka/ediff/HierarchicalActionSet.java | 2 +- .../serval/fixminer/jobs/AkkaTreeLoader.java | 82 +- .../serval/fixminer/jobs/EnhancedASTDiff.java | 18 +- .../serval/fixminer/jobs/ImportPairs2DB.java | 45 +- .../jobs/MultiThreadTreeLoaderCluster.java | 127 +- .../fixminer/jobs/StoreEDiffInCache.java | 71 +- .../lu/uni/serval/utils/ClusterToPattern.java | 71 + .../edu/lu/uni/serval/utils/EDiffHelper.java | 157 ++- 96 files changed, 3169 insertions(+), 190 deletions(-) create mode 100644 .idea/.name create mode 100644 .idea/compiler.xml create mode 100644 .idea/encodings.xml create mode 100644 .idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_2.xml create mode 100644 .idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_7.xml create mode 100644 .idea/libraries/Maven__ch_qos_logback_logback_classic_1_2_3.xml create mode 100644 .idea/libraries/Maven__ch_qos_logback_logback_core_1_1_2.xml create mode 100644 .idea/libraries/Maven__ch_qos_logback_logback_core_1_1_7.xml create mode 100644 .idea/libraries/Maven__ch_qos_logback_logback_core_1_2_3.xml create mode 100644 .idea/libraries/Maven__com_github_mpkorstanje_simmetrics_core_3_0_3.xml create mode 100644 .idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml create mode 100644 .idea/libraries/Maven__com_google_code_gson_gson_2_3.xml create mode 100644 .idea/libraries/Maven__com_google_guava_guava_11_0_2.xml create mode 100644 .idea/libraries/Maven__com_google_guava_guava_18_0.xml create mode 100644 .idea/libraries/Maven__com_nanohttpd_nanohttpd_2_1_1.xml create mode 100644 .idea/libraries/Maven__com_nanohttpd_nanohttpd_webserver_2_1_1.xml create mode 100644 .idea/libraries/Maven__com_rabbitmq_amqp_client_4_0_0.xml create mode 100644 .idea/libraries/Maven__com_sparkjava_spark_core_2_6_0.xml create mode 100644 .idea/libraries/Maven__com_typesafe_akka_akka_actor_2_11_2_4_11.xml create mode 100644 .idea/libraries/Maven__com_typesafe_config_1_3_0.xml create mode 100644 .idea/libraries/Maven__commons_codec_commons_codec_1_10.xml create mode 100644 .idea/libraries/Maven__commons_codec_commons_codec_1_9.xml create mode 100644 .idea/libraries/Maven__dom4j_dom4j_1_6_1.xml create mode 100644 .idea/libraries/Maven__javax_servlet_javax_servlet_api_3_1_0.xml create mode 100644 .idea/libraries/Maven__junit_junit_3_8_1.xml create mode 100644 .idea/libraries/Maven__junit_junit_4_11.xml create mode 100644 .idea/libraries/Maven__junit_junit_4_12.xml create mode 100644 .idea/libraries/Maven__log4j_log4j_1_2_14.xml create mode 100644 .idea/libraries/Maven__net_sf_trove4j_trove4j_3_0_3.xml create mode 100644 .idea/libraries/Maven__net_sourceforge_jexcelapi_jxl_2_6_12.xml create mode 100644 .idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml create mode 100644 .idea/libraries/Maven__org_apache_commons_commons_lang3_3_7.xml create mode 100644 .idea/libraries/Maven__org_apache_commons_commons_pool2_2_4_2.xml create mode 100644 .idea/libraries/Maven__org_apache_commons_commons_text_1_3.xml create mode 100644 .idea/libraries/Maven__org_apache_poi_poi_3_12.xml create mode 100644 .idea/libraries/Maven__org_apache_poi_poi_ooxml_3_12.xml create mode 100644 .idea/libraries/Maven__org_apache_poi_poi_ooxml_schemas_3_12.xml create mode 100644 .idea/libraries/Maven__org_apache_xmlbeans_xmlbeans_2_6_0.xml create mode 100644 .idea/libraries/Maven__org_eclipse_birt_runtime_org_eclipse_core_resources_3_10_0_v20150423_0755.xml create mode 100644 .idea/libraries/Maven__org_eclipse_core_contenttype_3_4_200_v20140207_1251.xml create mode 100644 .idea/libraries/Maven__org_eclipse_core_jobs_3_6_0_v20140424_0053.xml create mode 100644 .idea/libraries/Maven__org_eclipse_core_runtime_3_10_0_v20140318_2214.xml create mode 100644 .idea/libraries/Maven__org_eclipse_equinox_app_1_3_200_v20130910_1609.xml create mode 100644 .idea/libraries/Maven__org_eclipse_equinox_common_3_6_200_v20130402_1505.xml create mode 100644 .idea/libraries/Maven__org_eclipse_equinox_preferences_3_5_200_v20140224_1527.xml create mode 100644 .idea/libraries/Maven__org_eclipse_equinox_registry_3_5_400_v20140428_1507.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_jetty_client_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_jetty_http_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_jetty_io_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_jetty_security_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_jetty_server_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_jetty_servlet_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_jetty_util_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_jetty_webapp_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_jetty_xml_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_api_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_client_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_common_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_server_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_servlet_9_4_4_v20170414.xml create mode 100644 .idea/libraries/Maven__org_eclipse_osgi_3_10_0_v20140606_1445.xml create mode 100644 .idea/libraries/Maven__org_eclipse_tycho_org_eclipse_jdt_core_3_12_2_v20161117_1814.xml create mode 100644 .idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml create mode 100644 .idea/libraries/Maven__org_javassist_javassist_3_16_1_GA.xml create mode 100644 .idea/libraries/Maven__org_javatuples_javatuples_1_2.xml create mode 100644 .idea/libraries/Maven__org_reflections_reflections_0_9_9_RC1.xml create mode 100644 .idea/libraries/Maven__org_rendersnake_rendersnake_1_8.xml create mode 100644 .idea/libraries/Maven__org_scala_lang_modules_scala_java8_compat_2_11_0_7_0.xml create mode 100644 .idea/libraries/Maven__org_scala_lang_scala_library_2_11_8.xml create mode 100644 .idea/libraries/Maven__org_slf4j_slf4j_api_1_7_13.xml create mode 100644 .idea/libraries/Maven__org_slf4j_slf4j_api_1_7_21.xml create mode 100644 .idea/libraries/Maven__org_slf4j_slf4j_api_1_7_25.xml create mode 100644 .idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml create mode 100644 .idea/libraries/Maven__redis_clients_jedis_2_8_1.xml create mode 100644 .idea/libraries/Maven__stax_stax_api_1_0_1.xml create mode 100644 .idea/libraries/Maven__xml_apis_xml_apis_1_0_b2.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml create mode 100644 difffile create mode 100644 src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java create mode 100644 src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..8796191 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +FixPatternMiner \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..9d2856b --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..aad31c1 --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_2.xml b/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_2.xml new file mode 100644 index 0000000..a8b0d9c --- /dev/null +++ b/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_7.xml b/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_7.xml new file mode 100644 index 0000000..cdd7959 --- /dev/null +++ b/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_7.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_2_3.xml b/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_2_3.xml new file mode 100644 index 0000000..6fec8f4 --- /dev/null +++ b/.idea/libraries/Maven__ch_qos_logback_logback_classic_1_2_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_2.xml b/.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_2.xml new file mode 100644 index 0000000..1c76d0b --- /dev/null +++ b/.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_7.xml b/.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_7.xml new file mode 100644 index 0000000..6c2a760 --- /dev/null +++ b/.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_7.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__ch_qos_logback_logback_core_1_2_3.xml b/.idea/libraries/Maven__ch_qos_logback_logback_core_1_2_3.xml new file mode 100644 index 0000000..9eb8596 --- /dev/null +++ b/.idea/libraries/Maven__ch_qos_logback_logback_core_1_2_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_github_mpkorstanje_simmetrics_core_3_0_3.xml b/.idea/libraries/Maven__com_github_mpkorstanje_simmetrics_core_3_0_3.xml new file mode 100644 index 0000000..a009f87 --- /dev/null +++ b/.idea/libraries/Maven__com_github_mpkorstanje_simmetrics_core_3_0_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml b/.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml new file mode 100644 index 0000000..0e66824 --- /dev/null +++ b/.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_code_gson_gson_2_3.xml b/.idea/libraries/Maven__com_google_code_gson_gson_2_3.xml new file mode 100644 index 0000000..4dcaf15 --- /dev/null +++ b/.idea/libraries/Maven__com_google_code_gson_gson_2_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_guava_guava_11_0_2.xml b/.idea/libraries/Maven__com_google_guava_guava_11_0_2.xml new file mode 100644 index 0000000..01a573a --- /dev/null +++ b/.idea/libraries/Maven__com_google_guava_guava_11_0_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_guava_guava_18_0.xml b/.idea/libraries/Maven__com_google_guava_guava_18_0.xml new file mode 100644 index 0000000..bbd71d7 --- /dev/null +++ b/.idea/libraries/Maven__com_google_guava_guava_18_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_nanohttpd_nanohttpd_2_1_1.xml b/.idea/libraries/Maven__com_nanohttpd_nanohttpd_2_1_1.xml new file mode 100644 index 0000000..7c43448 --- /dev/null +++ b/.idea/libraries/Maven__com_nanohttpd_nanohttpd_2_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_nanohttpd_nanohttpd_webserver_2_1_1.xml b/.idea/libraries/Maven__com_nanohttpd_nanohttpd_webserver_2_1_1.xml new file mode 100644 index 0000000..c265679 --- /dev/null +++ b/.idea/libraries/Maven__com_nanohttpd_nanohttpd_webserver_2_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_rabbitmq_amqp_client_4_0_0.xml b/.idea/libraries/Maven__com_rabbitmq_amqp_client_4_0_0.xml new file mode 100644 index 0000000..0583b43 --- /dev/null +++ b/.idea/libraries/Maven__com_rabbitmq_amqp_client_4_0_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_sparkjava_spark_core_2_6_0.xml b/.idea/libraries/Maven__com_sparkjava_spark_core_2_6_0.xml new file mode 100644 index 0000000..a1b274c --- /dev/null +++ b/.idea/libraries/Maven__com_sparkjava_spark_core_2_6_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_typesafe_akka_akka_actor_2_11_2_4_11.xml b/.idea/libraries/Maven__com_typesafe_akka_akka_actor_2_11_2_4_11.xml new file mode 100644 index 0000000..09604f8 --- /dev/null +++ b/.idea/libraries/Maven__com_typesafe_akka_akka_actor_2_11_2_4_11.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_typesafe_config_1_3_0.xml b/.idea/libraries/Maven__com_typesafe_config_1_3_0.xml new file mode 100644 index 0000000..f92ac86 --- /dev/null +++ b/.idea/libraries/Maven__com_typesafe_config_1_3_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_codec_commons_codec_1_10.xml b/.idea/libraries/Maven__commons_codec_commons_codec_1_10.xml new file mode 100644 index 0000000..27424a1 --- /dev/null +++ b/.idea/libraries/Maven__commons_codec_commons_codec_1_10.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml b/.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml new file mode 100644 index 0000000..a66d039 --- /dev/null +++ b/.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__dom4j_dom4j_1_6_1.xml b/.idea/libraries/Maven__dom4j_dom4j_1_6_1.xml new file mode 100644 index 0000000..14681ee --- /dev/null +++ b/.idea/libraries/Maven__dom4j_dom4j_1_6_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__javax_servlet_javax_servlet_api_3_1_0.xml b/.idea/libraries/Maven__javax_servlet_javax_servlet_api_3_1_0.xml new file mode 100644 index 0000000..c24f7e3 --- /dev/null +++ b/.idea/libraries/Maven__javax_servlet_javax_servlet_api_3_1_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__junit_junit_3_8_1.xml b/.idea/libraries/Maven__junit_junit_3_8_1.xml new file mode 100644 index 0000000..71b2993 --- /dev/null +++ b/.idea/libraries/Maven__junit_junit_3_8_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__junit_junit_4_11.xml b/.idea/libraries/Maven__junit_junit_4_11.xml new file mode 100644 index 0000000..f33320d --- /dev/null +++ b/.idea/libraries/Maven__junit_junit_4_11.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__junit_junit_4_12.xml b/.idea/libraries/Maven__junit_junit_4_12.xml new file mode 100644 index 0000000..d411041 --- /dev/null +++ b/.idea/libraries/Maven__junit_junit_4_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__log4j_log4j_1_2_14.xml b/.idea/libraries/Maven__log4j_log4j_1_2_14.xml new file mode 100644 index 0000000..2825a67 --- /dev/null +++ b/.idea/libraries/Maven__log4j_log4j_1_2_14.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__net_sf_trove4j_trove4j_3_0_3.xml b/.idea/libraries/Maven__net_sf_trove4j_trove4j_3_0_3.xml new file mode 100644 index 0000000..72d4d7e --- /dev/null +++ b/.idea/libraries/Maven__net_sf_trove4j_trove4j_3_0_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__net_sourceforge_jexcelapi_jxl_2_6_12.xml b/.idea/libraries/Maven__net_sourceforge_jexcelapi_jxl_2_6_12.xml new file mode 100644 index 0000000..19e6af8 --- /dev/null +++ b/.idea/libraries/Maven__net_sourceforge_jexcelapi_jxl_2_6_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml new file mode 100644 index 0000000..32bfe3b --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_7.xml b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_7.xml new file mode 100644 index 0000000..28edf49 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_7.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_pool2_2_4_2.xml b/.idea/libraries/Maven__org_apache_commons_commons_pool2_2_4_2.xml new file mode 100644 index 0000000..2970b5e --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_pool2_2_4_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_text_1_3.xml b/.idea/libraries/Maven__org_apache_commons_commons_text_1_3.xml new file mode 100644 index 0000000..2930e62 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_text_1_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_poi_poi_3_12.xml b/.idea/libraries/Maven__org_apache_poi_poi_3_12.xml new file mode 100644 index 0000000..f3bfa63 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_poi_poi_3_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_poi_poi_ooxml_3_12.xml b/.idea/libraries/Maven__org_apache_poi_poi_ooxml_3_12.xml new file mode 100644 index 0000000..0f92377 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_poi_poi_ooxml_3_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_poi_poi_ooxml_schemas_3_12.xml b/.idea/libraries/Maven__org_apache_poi_poi_ooxml_schemas_3_12.xml new file mode 100644 index 0000000..fb871da --- /dev/null +++ b/.idea/libraries/Maven__org_apache_poi_poi_ooxml_schemas_3_12.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_xmlbeans_xmlbeans_2_6_0.xml b/.idea/libraries/Maven__org_apache_xmlbeans_xmlbeans_2_6_0.xml new file mode 100644 index 0000000..cb0f76d --- /dev/null +++ b/.idea/libraries/Maven__org_apache_xmlbeans_xmlbeans_2_6_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_birt_runtime_org_eclipse_core_resources_3_10_0_v20150423_0755.xml b/.idea/libraries/Maven__org_eclipse_birt_runtime_org_eclipse_core_resources_3_10_0_v20150423_0755.xml new file mode 100644 index 0000000..bec40e7 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_birt_runtime_org_eclipse_core_resources_3_10_0_v20150423_0755.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_core_contenttype_3_4_200_v20140207_1251.xml b/.idea/libraries/Maven__org_eclipse_core_contenttype_3_4_200_v20140207_1251.xml new file mode 100644 index 0000000..f23ea73 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_core_contenttype_3_4_200_v20140207_1251.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_core_jobs_3_6_0_v20140424_0053.xml b/.idea/libraries/Maven__org_eclipse_core_jobs_3_6_0_v20140424_0053.xml new file mode 100644 index 0000000..ecd0a0f --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_core_jobs_3_6_0_v20140424_0053.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_core_runtime_3_10_0_v20140318_2214.xml b/.idea/libraries/Maven__org_eclipse_core_runtime_3_10_0_v20140318_2214.xml new file mode 100644 index 0000000..b1c50eb --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_core_runtime_3_10_0_v20140318_2214.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_equinox_app_1_3_200_v20130910_1609.xml b/.idea/libraries/Maven__org_eclipse_equinox_app_1_3_200_v20130910_1609.xml new file mode 100644 index 0000000..c5372f9 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_equinox_app_1_3_200_v20130910_1609.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_equinox_common_3_6_200_v20130402_1505.xml b/.idea/libraries/Maven__org_eclipse_equinox_common_3_6_200_v20130402_1505.xml new file mode 100644 index 0000000..0e8e398 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_equinox_common_3_6_200_v20130402_1505.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_equinox_preferences_3_5_200_v20140224_1527.xml b/.idea/libraries/Maven__org_eclipse_equinox_preferences_3_5_200_v20140224_1527.xml new file mode 100644 index 0000000..cf2ff45 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_equinox_preferences_3_5_200_v20140224_1527.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_equinox_registry_3_5_400_v20140428_1507.xml b/.idea/libraries/Maven__org_eclipse_equinox_registry_3_5_400_v20140428_1507.xml new file mode 100644 index 0000000..a35d91c --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_equinox_registry_3_5_400_v20140428_1507.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_client_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_client_9_4_4_v20170414.xml new file mode 100644 index 0000000..753cd46 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_client_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_http_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_http_9_4_4_v20170414.xml new file mode 100644 index 0000000..24b9595 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_http_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_io_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_io_9_4_4_v20170414.xml new file mode 100644 index 0000000..be4af0a --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_io_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_security_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_security_9_4_4_v20170414.xml new file mode 100644 index 0000000..0a0a61a --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_security_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_server_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_server_9_4_4_v20170414.xml new file mode 100644 index 0000000..aa28eb9 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_server_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_servlet_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_servlet_9_4_4_v20170414.xml new file mode 100644 index 0000000..8923ab4 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_servlet_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_util_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_util_9_4_4_v20170414.xml new file mode 100644 index 0000000..b7d325c --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_util_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_webapp_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_webapp_9_4_4_v20170414.xml new file mode 100644 index 0000000..bd09afa --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_webapp_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_jetty_xml_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_jetty_xml_9_4_4_v20170414.xml new file mode 100644 index 0000000..9f8b202 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_jetty_xml_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_api_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_api_9_4_4_v20170414.xml new file mode 100644 index 0000000..597060a --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_api_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_client_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_client_9_4_4_v20170414.xml new file mode 100644 index 0000000..be2f9c9 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_client_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_common_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_common_9_4_4_v20170414.xml new file mode 100644 index 0000000..af4b520 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_common_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_server_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_server_9_4_4_v20170414.xml new file mode 100644 index 0000000..26b182a --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_server_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_servlet_9_4_4_v20170414.xml b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_servlet_9_4_4_v20170414.xml new file mode 100644 index 0000000..d11dc2b --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_jetty_websocket_websocket_servlet_9_4_4_v20170414.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_osgi_3_10_0_v20140606_1445.xml b/.idea/libraries/Maven__org_eclipse_osgi_3_10_0_v20140606_1445.xml new file mode 100644 index 0000000..4f8664b --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_osgi_3_10_0_v20140606_1445.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_eclipse_tycho_org_eclipse_jdt_core_3_12_2_v20161117_1814.xml b/.idea/libraries/Maven__org_eclipse_tycho_org_eclipse_jdt_core_3_12_2_v20161117_1814.xml new file mode 100644 index 0000000..152bde1 --- /dev/null +++ b/.idea/libraries/Maven__org_eclipse_tycho_org_eclipse_jdt_core_3_12_2_v20161117_1814.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml b/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml new file mode 100644 index 0000000..f58bbc1 --- /dev/null +++ b/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_javassist_javassist_3_16_1_GA.xml b/.idea/libraries/Maven__org_javassist_javassist_3_16_1_GA.xml new file mode 100644 index 0000000..7b4e548 --- /dev/null +++ b/.idea/libraries/Maven__org_javassist_javassist_3_16_1_GA.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_javatuples_javatuples_1_2.xml b/.idea/libraries/Maven__org_javatuples_javatuples_1_2.xml new file mode 100644 index 0000000..b524531 --- /dev/null +++ b/.idea/libraries/Maven__org_javatuples_javatuples_1_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_reflections_reflections_0_9_9_RC1.xml b/.idea/libraries/Maven__org_reflections_reflections_0_9_9_RC1.xml new file mode 100644 index 0000000..95dd64a --- /dev/null +++ b/.idea/libraries/Maven__org_reflections_reflections_0_9_9_RC1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_rendersnake_rendersnake_1_8.xml b/.idea/libraries/Maven__org_rendersnake_rendersnake_1_8.xml new file mode 100644 index 0000000..7251aaa --- /dev/null +++ b/.idea/libraries/Maven__org_rendersnake_rendersnake_1_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_scala_lang_modules_scala_java8_compat_2_11_0_7_0.xml b/.idea/libraries/Maven__org_scala_lang_modules_scala_java8_compat_2_11_0_7_0.xml new file mode 100644 index 0000000..f4b289b --- /dev/null +++ b/.idea/libraries/Maven__org_scala_lang_modules_scala_java8_compat_2_11_0_7_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_8.xml b/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_8.xml new file mode 100644 index 0000000..fc8460c --- /dev/null +++ b/.idea/libraries/Maven__org_scala_lang_scala_library_2_11_8.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_13.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_13.xml new file mode 100644 index 0000000..4fd913d --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_13.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_21.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_21.xml new file mode 100644 index 0000000..1b644dd --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_21.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_25.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_25.xml new file mode 100644 index 0000000..20e8163 --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_25.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml new file mode 100644 index 0000000..1e67260 --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__redis_clients_jedis_2_8_1.xml b/.idea/libraries/Maven__redis_clients_jedis_2_8_1.xml new file mode 100644 index 0000000..f12195b --- /dev/null +++ b/.idea/libraries/Maven__redis_clients_jedis_2_8_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__stax_stax_api_1_0_1.xml b/.idea/libraries/Maven__stax_stax_api_1_0_1.xml new file mode 100644 index 0000000..0b13335 --- /dev/null +++ b/.idea/libraries/Maven__stax_stax_api_1_0_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__xml_apis_xml_apis_1_0_b2.xml b/.idea/libraries/Maven__xml_apis_xml_apis_1_0_b2.xml new file mode 100644 index 0000000..c36e717 --- /dev/null +++ b/.idea/libraries/Maven__xml_apis_xml_apis_1_0_b2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..cc1f0b7 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,15 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..272b874 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..dc408e4 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..19f4360 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,1218 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileHelper + contain + Time + getSub + Filenamef + .keys + write + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1550924800456 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 25 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 83 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java + 26 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java + 33 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 77 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 80 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java + 27 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java + 79 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java + 77 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java + 37 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java + 94 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java + 104 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 72 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java + 85 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 84 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java + 72 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java + 34 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java + 54 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java + 33 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java + 84 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java + 23 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java + 116 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java + 47 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java + 140 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java + 78 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java + 114 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java + 131 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java + 184 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java + 61 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java + 28 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java + 35 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java + 41 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java + 56 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java + 98 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java + 141 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java + 114 + + + + file://$PROJECT_DIR$/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java + 146 + + + + + + + + + folder.getParent() + JAVA + CODE_FRAGMENT + + + shape.getParent() + JAVA + CODE_FRAGMENT + + + cluster.getName() + JAVA + CODE_FRAGMENT + + + shape.getPath() + JAVA + CODE_FRAGMENT + + + cluster.getPath() + JAVA + CODE_FRAGMENT + + + actionSet.toString() + JAVA + CODE_FRAGMENT + + + file.getName() + JAVA + CODE_FRAGMENT + + + file. + JAVA + CODE_FRAGMENT + + + inner.get(prefix+"-"+firstValue); + JAVA + CODE_FRAGMENT + + + + + 1 + JAVA + EXPRESSION + + + "JDT" + JAVA + EXPRESSION + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + No facets are configured + + + + + + + + + + + + + + + 1.8 + + + + + + + + client + + + + + + + + 1.8 + + + + + + + + Maven: ch.qos.logback:logback-classic:1.1.2 + + + + + + + + \ No newline at end of file diff --git a/FixPatternMiner.iml b/FixPatternMiner.iml index a7d8955..82122fc 100644 --- a/FixPatternMiner.iml +++ b/FixPatternMiner.iml @@ -1,6 +1,6 @@ - + @@ -10,7 +10,17 @@ + + + + + + + + + + @@ -18,18 +28,9 @@ - - - - - - - - - diff --git a/difffile b/difffile new file mode 100644 index 0000000..4dc952b --- /dev/null +++ b/difffile @@ -0,0 +1,210 @@ +diff --git a/.gitignore b/.gitignore +index 24cae48..27c7ba3 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -14,6 +14,7 @@ GumTreeInput/ + GumTreeInput1/ + GumTreeOutput1/ + GumTreeResults/ ++.idea/* + + */Test.java + *TestParser.java +diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +index 4d885ea..eb1eca1 100644 +--- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java ++++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +@@ -1,5 +1,9 @@ + package edu.lu.uni.serval.FixPatternParser.violations; + ++import com.github.gumtreediff.actions.model.Delete; ++import com.github.gumtreediff.actions.model.Insert; ++import com.github.gumtreediff.actions.model.Move; ++import com.github.gumtreediff.actions.model.Update; + import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; + + import java.io.*; +@@ -30,27 +34,29 @@ public class FixedViolationHunkParser extends FixedViolationParser { + public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project) { + List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); + if (actionSets.size() != 0) { +-// boolean isUpdate = +-// actionSets.stream().allMatch(p -> p.getAction() instanceof Update); +-// boolean isInsert = +-// actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); +-// boolean isDelete = +-// actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); +-// boolean isMove = +-// actionSets.stream().allMatch(p -> p.getAction() instanceof Move); + int hunkSet = 0; + // if (isUpdate || isInsert || isDelete || isMove) { + for (HierarchicalActionSet actionSet : actionSets) { +- String folder = "/ALL/"; +-// if (isUpdate) { +-// folder = "/UPD/"; +-// } else if (isDelete) { +-// folder = "/DEL/"; +-// } else if (isInsert) { +-// folder = "/INS/"; +-// } else if (isMove) { +-// folder = "/MOV/"; +-// } ++ boolean isUpdate = ++ actionSets.stream().allMatch(p -> p.getAction() instanceof Update); ++ boolean isInsert = ++ actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); ++ boolean isDelete = ++ actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); ++ boolean isMove = ++ actionSets.stream().allMatch(p -> p.getAction() instanceof Move); ++ String folder = "/MIX/"; ++ if (isUpdate) { ++ folder = "/UPD/"; ++ } else if (isDelete) { ++ folder = "/DEL/"; ++ } else if (isInsert) { ++ folder = "/INS/"; ++ } else if (isMove) { ++ folder = "/MOV/"; ++ } else{ ++ folder = "/MIX/"; ++ } + + + FileOutputStream f = null; +diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java +index 3084dbe..ffe1b01 100644 +--- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java ++++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java +@@ -61,7 +61,7 @@ public class ParseFixPatternWorker extends UntypedActor { + int testInfos = 0; + int timeouts = 0; + StringBuilder builder = new StringBuilder(); +- ++// log.info("Work queue size " + files.size()); + for (MessageFile msgFile : files) { + File revFile = msgFile.getRevFile(); + File prevFile = msgFile.getPrevFile(); +diff --git a/src/main/java/edu/lu/uni/serval/config/Configuration.java b/src/main/java/edu/lu/uni/serval/config/Configuration.java +index 4421c72..7162180 100644 +--- a/src/main/java/edu/lu/uni/serval/config/Configuration.java ++++ b/src/main/java/edu/lu/uni/serval/config/Configuration.java +@@ -2,6 +2,6 @@ package edu.lu.uni.serval.config; + + public class Configuration { + +- public static final long SECONDS_TO_WAIT = 900L; ++ public static final long SECONDS_TO_WAIT = 90000L; + + } +diff --git a/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java +index 43895e0..066b07d 100644 +--- a/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java ++++ b/src/main/java/edu/lu/uni/serval/fixminer/EnhancedASTDiff.java +@@ -5,7 +5,7 @@ import akka.actor.ActorSystem; + import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; + import edu.lu.uni.serval.MultipleThreadsParser.ParseFixPatternActor; + import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage; +-import edu.lu.uni.serval.utils.FileHelper; ++import edu.lu.uni.serval.FixPattern.utils.FileHelper; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + +@@ -13,6 +13,7 @@ import java.io.File; + import java.util.ArrayList; + import java.util.Arrays; + import java.util.List; ++import java.util.Locale; + import java.util.stream.Collectors; + import java.util.stream.Stream; + +@@ -31,6 +32,7 @@ public class EnhancedASTDiff { + Stream stream = Arrays.stream(listOfFiles); + List folders = stream + .filter(x -> !x.getName().startsWith(".")) ++// .filter(x -> x.getName().startsWith("JDT")) + .collect(Collectors.toList()); + + +@@ -49,11 +51,11 @@ public class EnhancedASTDiff { + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); +- FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ALL"); ++ FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MIX"); + + + int a = 0; +- ++ log.info("Work queue size " + msgFiles.size()); + ActorSystem system = null; + ActorRef parsingActor = null; + final WorkMessage msg = new WorkMessage(0, msgFiles); +@@ -80,6 +82,11 @@ public class EnhancedASTDiff { + if (revFiles.length >= 0) { + for (File revFile : revFiles) { + String fileName = revFile.getName(); ++ if(fileName.toLowerCase(Locale.ENGLISH).contains("test")){ ++ log.info(fileName); ++ continue; ++ } ++ + File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file + fileName = fileName.replace(".java", ".txt"); + File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file +diff --git a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java +index 44575a7..88f36c6 100644 +--- a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java ++++ b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java +@@ -5,7 +5,7 @@ import akka.actor.ActorSystem; + import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; + import edu.lu.uni.serval.MultipleThreadsParser.ParseFixPatternActor; + import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage; +-import edu.lu.uni.serval.utils.FileHelper; ++import edu.lu.uni.serval.FixPattern.utils.FileHelper; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + +diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java +index 9c1d05d..3dceda0 100644 +--- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java ++++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java +@@ -37,7 +37,7 @@ public class CallShell { + } + // Thread.sleep(Integer.valueOf(serverWait)); + +- String cmd = "redis-cli -p %s ping"; ++ String cmd = "bash source activate redisEnv && redis-cli -p %s ping"; + String cmd1 = String.format(cmd,Integer.valueOf(port)); + runPing(cmd1); + +diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java +index 39bba51..8657dce 100644 +--- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java ++++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreEDiffInCache.java +@@ -52,13 +52,15 @@ public class StoreEDiffInCache { + .filter(x -> x.getName().startsWith("UPD") || + x.getName().startsWith("INS") || + x.getName().startsWith("DEL") || ++ x.getName().startsWith("MIX") || + x.getName().startsWith("MOV")) + .collect(Collectors.toList()); + File[] files1 = fs.get(0).listFiles(); + File[] files2 = fs.get(1).listFiles(); + File[] files3 = fs.get(2).listFiles(); + File[] files4 = fs.get(3).listFiles(); +- dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new); ++ File[] files5 = fs.get(4).listFiles(); ++ dumps = Stream.of(files1, files2, files3,files4,files5).flatMap(Stream::of).toArray(File[]::new); + }else{ + fs = fileStream + .filter(x -> x.getName().startsWith(operation)) +@@ -70,7 +72,7 @@ public class StoreEDiffInCache { + for (File f : dumps) { + String name = f.getName(); + +- String key = pjName + "/"+ operation+"/" + name; ++ String key = pjName + f.getPath().replace(pj.getPath(),""); + String result = key +","+f.getPath(); + workList.add(result); + } diff --git a/pom.xml b/pom.xml index 20628f7..e783f0d 100644 --- a/pom.xml +++ b/pom.xml @@ -25,11 +25,11 @@ 1.2 - - - - - + + edu.lu.uni + simple-utils + 0.0.1-SNAPSHOT + @@ -135,7 +135,7 @@ - edu.lu.uni.serval.fixminer.Launcher + edu.lu.uni.serval.fixminer.akka.compare.CompareTrees diff --git a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java index a4cbbd0..79e1c22 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java @@ -68,7 +68,7 @@ public class Launcher { gumInput = datasetPath +"/"+pjName+"/"; gumOutput = datasetPath + "/EnhancedASTDiff" + pjName; dbDir = datasetPath + "/redis"; - pairsPath = datasetPath + "/pairsImport"+pjName; + pairsPath = datasetPath + "/pairs"; dumpsName = "dumps-"+pjName+".rdb"; int iCursor = Integer.valueOf(cursor); @@ -81,11 +81,11 @@ public class Launcher { StoreEDiffInCache.main(gumOutput, portDumps, dbDir, actionType+dumpsName,actionType); break; case "SI": - CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType,isBigPair,iCursor); - ImportPairs2DB.main(pairsPath+actionType, portInner, dbDir,datasetPath,chunk); +// CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType,isBigPair,iCursor); + ImportPairs2DB.main(pairsPath, portInner, dbDir,datasetPath,chunk); break; case "SIMI": - AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,iCursor,chunk,eDiffTimeout,parallelism); + AkkaTreeLoader.main(portInner, dbDir, "pairs.rdb" , portDumps, actionType+dumpsName,pairsPath,numOfWorkers,iCursor,chunk,eDiffTimeout,parallelism); break; case "LEVEL1": @@ -136,9 +136,9 @@ public class Launcher { private static void level2(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput,int cursor) throws Exception { String stopServer; - MultiThreadTreeLoaderCluster.calculatePairsOfClusters(datasetPath + "/cluster"+pjName+ actionType, datasetPath,actionType); +// MultiThreadTreeLoaderCluster.calculatePairsOfClusters(datasetPath + "/cluster"+pjName+ actionType, datasetPath,actionType); - MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath+"/pairs"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType,cursor); + MultiThreadTreeLoaderCluster.mainCompare(port, datasetPath+"/pairsAction", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-"+pjName+actionType+".rdb", actionType+dumpsName, "6380",actionType,cursor); CallShell cs3 =new CallShell(); String db22 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java index 15cbc87..6a23292 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java @@ -87,7 +87,8 @@ public class AkkaTreeParser { log.info("Scanning "); sc.count(cursor); - sc.match("pair_[0-9]*"); + sc.match("*"); +// sc.match("pair_[0-9]*"); scan = inner.scan("0", sc); int size = scan.getResult().size(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java index ef93559..911d50b 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java @@ -38,11 +38,12 @@ public class Compare { String i = split[1]; String j = split[2]; + String keyName = split[0]; - oldTree = EDiffHelper.getSimpliedTree(i,outerPool); + oldTree = EDiffHelper.getSimpliedTree(keyName,i,outerPool); - newTree = EDiffHelper.getSimpliedTree(j,outerPool); + newTree = EDiffHelper.getSimpliedTree(keyName,j,outerPool); Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); m.match(); @@ -66,7 +67,7 @@ public class Compare { if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { - String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); + String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); log.info("{} tagged to be similar" ,matchKey); jedis.select(1); @@ -76,7 +77,7 @@ public class Compare { jedis.select(0); - jedis.del("pair_" + (String.valueOf(i)) + "_" + String.valueOf(j)); + jedis.del(keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j)); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java new file mode 100644 index 0000000..510230e --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java @@ -0,0 +1,197 @@ +package edu.lu.uni.serval.fixminer.akka.compare; + +import com.github.gumtreediff.actions.ActionGenerator; +import com.github.gumtreediff.actions.model.Action; +import com.github.gumtreediff.matchers.Matcher; +import com.github.gumtreediff.matchers.Matchers; +import com.github.gumtreediff.tree.ITree; +import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.PoolBuilder; +import org.apache.commons.text.similarity.JaroWinklerDistance; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static edu.lu.uni.serval.fixminer.jobs.MultiThreadTreeLoaderCluster3.getNames; + + +/** + * Created by anilkoyuncu on 03/04/2018. + */ +public class CompareTrees { + + private static Logger log = LoggerFactory.getLogger(CompareTrees.class); + + + public static void main(String[] args) throws IOException { + + String portInner = "6380"; + String port = "6399"; +// + JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000); + + JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000); + + List listOfPairs = AkkaTreeParser.getMessages(innerPool,100000000); +// listOfPairs.stream().parallel(). + + int counter = new Object() { + int counter = 0; + + { + + listOfPairs.parallelStream(). + peek(x -> counter++). + forEach(m -> + { +// Compare compare = new Compare(); + coreCompare(m, args[0],innerPool, outerPool); + if (counter % 1000 == 0) { + log.info("Finalized parsing " + counter + " files... remaing " + (listOfPairs.size() - counter)); + } + } + ); + } + }.counter; +// coreCompare(args[0],args[1],args[2],args[3]); + } + public static void coreCompare(String pairName, String treeType,JedisPool innerPool,JedisPool outerPool ) { + + +// String portInner = "6380"; +// String port = "6399"; + +// JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000); +// +// JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000); + + Map resultMap; + Jedis jedis = null; + ITree oldTree = null; + ITree newTree = null; + + try { + jedis = innerPool.getResource(); + + String[] split = pairName.split("_"); + + + String i = split[1]; + String j = split[2]; + String keyName = split[0]; + + switch (treeType) { + case "shape": + oldTree = EDiffHelper.getShapes(keyName, i, outerPool); + newTree = EDiffHelper.getShapes(keyName, j, outerPool); + break; + case "action": + oldTree = EDiffHelper.getActions(keyName, i, outerPool,innerPool); + newTree = EDiffHelper.getActions(keyName, j, outerPool,innerPool); + break; + case "token": + oldTree = EDiffHelper.getTokens(keyName, i, outerPool,innerPool); + newTree = EDiffHelper.getTokens(keyName, j, outerPool,innerPool); + List oldTokens = new ArrayList<>(); + List newTokens = new ArrayList<>(); + + + + + oldTokens = getNames(oldTree,oldTokens); + newTokens = getNames(newTree,newTokens); + + + CharSequence[] oldSequences = oldTokens.toArray(new CharSequence[oldTokens.size()]); + CharSequence[] newSequences = newTokens.toArray(new CharSequence[newTokens.size()]); + JaroWinklerDistance jwd = new JaroWinklerDistance(); + + Double overallSimi = Double.valueOf(0); + if(oldSequences.length > 0 && (oldSequences.length == newSequences.length)){ + for (int idx = 0; idx < newSequences.length; idx++) { + Double simi = jwd.apply(newSequences[idx], oldSequences[idx]); + overallSimi = simi + overallSimi; + } + overallSimi = overallSimi / oldSequences.length; + }else{ + overallSimi = Double.valueOf(0); +// if(oldSequences.length != 0) { +// log.info("ERROR"); +// } + } + + int retval = Double.compare(overallSimi, Double.valueOf(0.8)); + String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); + if(retval >= 0){ + + + + String result = i + "," + j + ","+String.join(",", oldTokens); + jedis.select(2); + jedis.set(matchKey, result); + } + jedis.select(0); + jedis.del(matchKey); + + return; + default: + break; + } + + Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); + m.match(); + + + ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); + ag.generate(); + List actions = ag.getActions(); + + double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree); + String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1); + double diceSimilarity1 = m.diceSimilarity(oldTree, newTree); + String diceSimilarity = String.format("%1.2f", diceSimilarity1); + double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree); + String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); + + String editDistance = String.valueOf(actions.size()); + + String result = i + "," + j + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; + + String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); + + if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) + || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { +// log.info("{} tagged to be similar" ,matchKey); + + jedis.select(2); + jedis.set(matchKey, result); + + } + + + jedis.select(0); + jedis.del(matchKey); + + + + } catch (Exception e) { + + log.debug("{} not comparable", pairName); +// e.printStackTrace(); + + + }finally { + if (jedis != null) { + jedis.close(); + } + } + } + + +} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java index c2b6309..667680b 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java @@ -4,6 +4,7 @@ import com.github.gumtreediff.actions.model.Delete; import com.github.gumtreediff.actions.model.Insert; import com.github.gumtreediff.actions.model.Move; import com.github.gumtreediff.actions.model.Update; +import edu.lu.uni.serval.utils.FileHelper; import java.io.*; import java.util.List; @@ -23,41 +24,41 @@ public class EDiffHunkParser extends EDiffParser { public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project,String actionType) { List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); if (actionSets.size() != 0) { - String folder= null; - boolean processActionSet = false; +// String folder= null; + boolean processActionSet = true; // switch (actionType){ // case "ALL": - if(actionType.equals("ALL")){ - folder = "/ALL/"; - processActionSet = true; - }else if(actionType.equals("UPD") || actionType.equals("INS") || actionType.equals("DEL") || actionType.equals("MOV")|| actionType.equals("MIX")){ - boolean isUPD = actionSets.stream().allMatch(p -> p.getAction() instanceof Update); - boolean isINS = actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); - boolean isDEL = actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); - boolean isMOV = actionSets.stream().allMatch(p -> p.getAction() instanceof Move); - if(isUPD){ - folder = "/UPD/"; - processActionSet = true; - }else if(isINS){ - folder = "/INS/"; - processActionSet = true; - }else if(isDEL){ - folder = "/DEL/"; - processActionSet = true; - }else if(isMOV){ - folder = "/MOV/"; - processActionSet = true; - }else{ - folder = "/MIX/"; - processActionSet = true; - } - }else{ - - - processActionSet = false; - System.err.print(actionType + "not known"); - - } +// if(actionType.equals("ALL")){ +// folder = "/ALL/"; +// processActionSet = true; +// }else if(actionType.equals("UPD") || actionType.equals("INS") || actionType.equals("DEL") || actionType.equals("MOV")|| actionType.equals("MIX")){ +// boolean isUPD = actionSets.stream().allMatch(p -> p.getAction() instanceof Update); +// boolean isINS = actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); +// boolean isDEL = actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); +// boolean isMOV = actionSets.stream().allMatch(p -> p.getAction() instanceof Move); +// if(isUPD){ +// folder = "/UPD/"; +// processActionSet = true; +// }else if(isINS){ +// folder = "/INS/"; +// processActionSet = true; +// }else if(isDEL){ +// folder = "/DEL/"; +// processActionSet = true; +// }else if(isMOV){ +// folder = "/MOV/"; +// processActionSet = true; +// }else{ +// folder = "/MIX/"; +// processActionSet = true; +// } +// }else{ +// +// +// processActionSet = false; +// System.err.print(actionType + "not known"); +// +// } int hunkSet = 0; @@ -70,12 +71,21 @@ public class EDiffHunkParser extends EDiffParser { FileOutputStream f = null; try { + + String astNodeType = actionSet.getAstNodeType(); + int size = actionSet.toString().split("\\n").length; + +// int size = actionSet.strList.size(); + String datasetName = project; String[] split1 = diffentryFile.getParent().split(datasetName); String root = split1[0]; String pj = split1[1].split("/")[1]; - String hunkTreeFileName = root + "EnhancedASTDiff" + datasetName + "/" + pj + folder + diffentryFile.getName() + "_" + String.valueOf(hunkSet); + File file = new File(root + "EnhancedASTDiff" + datasetName + "/"+astNodeType+"/"+String.valueOf(size)+"/"); + file.mkdirs(); + String hunkTreeFileName = root + "EnhancedASTDiff" + datasetName + "/"+astNodeType+"/"+String.valueOf(size)+"/" + pj +"_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet); +// String hunkTreeFileName = root + "EnhancedASTDiff" + datasetName + "/" + pj + folder + diffentryFile.getName() + "_" + String.valueOf(hunkSet); f = new FileOutputStream(new File(hunkTreeFileName)); ObjectOutputStream o = new ObjectOutputStream(f); o.writeObject(actionSet); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java index 3a019e0..62f44e4 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java @@ -168,7 +168,7 @@ public class HierarchicalActionSet implements Comparable, return this.startPosition.compareTo(o.startPosition);//this.action.compareTo(o.action); } - + private List strList = new ArrayList<>(); @Override diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java index e154673..915e4bc 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java @@ -8,11 +8,16 @@ import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import java.io.BufferedReader; +import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser.akkaCompare; @@ -47,44 +52,63 @@ public class AkkaTreeLoader { - String pairsIndexFile = pairsPath + "/"+ chunkName; +// String pairsIndexFile = pairsPath + "/"+ chunkName; //0.txt.rdb - pairsIndexFile = pairsIndexFile.replace(chunk+".rdb",".index"); +// pairsIndexFile = pairsIndexFile.replace(chunk+".rdb",".index"); + File folder = new File(pairsPath); + File[] rootsFolder = folder.listFiles(); - Pattern pattern = Pattern.compile(","); - String csvFile = pairsIndexFile; - try { - try (BufferedReader in = new BufferedReader(new FileReader(csvFile));){ - Map namefreq = in - .lines() - .map(x -> pattern.split(x)) - .collect(HashMap::new, (map, x) -> - map.put(x[0], x[1]), - Map::putAll); + Stream stream = Arrays.stream(rootsFolder); + List roots = stream + .filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); - Jedis inner = null; + for(File root:roots) { + + File[] files = root.listFiles(); + Stream fileStream = Arrays.stream(files); + List pairsIndexFiles = fileStream + .filter(x -> !x.getName().startsWith(".")) + .filter(x -> x.getName().endsWith(".index")) + .collect(Collectors.toList()); + for (File pairsIndexFile : pairsIndexFiles) { + + Pattern pattern = Pattern.compile(","); + String csvFile = pairsIndexFile.getPath(); try { - inner = outerPool.getResource(); + try (BufferedReader in = new BufferedReader(new FileReader(csvFile));) { + Map namefreq = in + .lines() + .map(x -> pattern.split(x)) + .collect(HashMap::new, (map, x) -> + map.put(root.getName()+"-"+pairsIndexFile.getName().split("\\.")[0]+"-"+x[0], x[1]), + Map::putAll); + + Jedis inner = null; + try { + inner = outerPool.getResource(); + + for (Map.Entry entry : namefreq.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + inner.select(1); + inner.set(key, value); + } + + + } finally { + if (inner != null) { + inner.close(); + } + } + - for (Map.Entry entry : namefreq.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - inner.select(1); - inner.set(key,value); - } - - - }finally { - if (inner != null) { - inner.close(); } + } catch (IOException e) { + e.printStackTrace(); } - - } - } catch (IOException e) { - e.printStackTrace(); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java index afe421b..f2d3c44 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java @@ -50,15 +50,15 @@ public class EnhancedASTDiff { // a - if (actionType.equals("ALL")) { - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/" + actionType); - } else { - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MIX"); - } +// if (actionType.equals("ALL")) { +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/" + actionType); +// } else { +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MIX"); +// } } switch (parallelism){ diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java index e3a04c7..585d082 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java @@ -30,36 +30,49 @@ public class ImportPairs2DB { File folder = new File(csvInputPath); File[] subFolders = folder.listFiles(); Stream stream = Arrays.stream(subFolders); - List pjs = stream - .filter(x -> x.getName().endsWith(chunkType)) + List roots = stream +// .filter(x -> x.getName().endsWith(chunkType)) .collect(Collectors.toList()); + Integer portInt = Integer.valueOf(portInner); - for (File pj : pjs) { + String cmd = "bash " + dbDir + "/" + "startServer.sh" + " %s %s %s"; + cmd = String.format(cmd, dbDir,"pairs.rdb", portInt); + log.info(cmd); + CallShell cs = new CallShell(); + cs.runShell(cmd, portInner); - String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; - cmd = String.format(cmd, dbDir,pj.getName() +".rdb", portInt); - log.info(cmd); - CallShell cs = new CallShell(); - cs.runShell(cmd, portInner); + for (File root : roots) { - cmd = "bash "+datasetPath + "/redisSingleImport.sh" +" %s %s"; + File[] files = root.listFiles(); + Stream fileStream = Arrays.stream(files); + List pairs = fileStream + .filter(x -> x.getName().endsWith(chunkType)) + .collect(Collectors.toList()); - cmd = String.format(cmd, pj.getPath(), portInt); - log.info(cmd); - cs.runShell(cmd,portInner); + for(File pj:pairs) { - String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; - String stopServer2 = String.format(stopServer,portInt); - cs.runShell(stopServer2, portInner); - portInt++; + + cmd = "bash " + datasetPath + "/redisSingleImport.sh" + " %s %s %s"; + + cmd = String.format(cmd, pj.getPath(), portInt,root.getName()+"-"+pj.getName().split("\\.")[0]); + + log.info(cmd); + cs.runShell(cmd,portInner); + + + } } + String stopServer = "bash " + dbDir + "/" + "stopServer.sh" + " %s"; + String stopServer2 = String.format(stopServer, portInt); + cs.runShell(stopServer2, portInner); +// portInt++; log.info(parameters); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java index 9b993ad..84e487d 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java @@ -20,10 +20,8 @@ import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanResult; import java.io.*; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -49,8 +47,8 @@ public class MultiThreadTreeLoaderCluster { - String cmd3; - cmd3 = "bash " + importScript +" %s %s"; +// String cmd3; +// cmd3 = "bash " + importScript +" %s %s"; JedisPool jedisPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000); @@ -65,41 +63,74 @@ public class MultiThreadTreeLoaderCluster { .filter(x -> !x.getName().startsWith(".")) .collect(Collectors.toList()); - for (File f:folders){ + for (File f:folders) { // if(f.getName().startsWith("cluster0")) { + File[] files = f.listFiles(); + Stream fileStream = Arrays.stream(files); + + List pairs = fileStream + .filter(x -> !x.getName().startsWith(".")) + .filter(x -> !x.getName().endsWith(".index")) + .collect(Collectors.toList()); + for (File pair : pairs) { try (Jedis jedis = jedisPool.getResource()) { // do operations with jedis resource ScanParams sc = new ScanParams(); sc.count(cursor); - sc.match("pair_[0-9]*"); + sc.match(f.getName()+"*"); log.info("Scanning"); ScanResult scan = jedis.scan("0", sc); int size = scan.getResult().size(); if (size == 0) { - String comd = String.format(cmd3,f.getPath(),portInner); + String cmd3 = "bash " + importScript + " %s %s %s"; + + cmd3 = String.format(cmd3, pair.getPath(), portInner,f.getName()+"-"+pair.getName().split("\\.")[0]); +// String comd = String.format(cmd3, f.getPath(), portInner); // loadRedis(comd); - log.info("Importing {} pairs for cluster {}",size,f.getName()); - cs.runShell(comd,portInner); + log.info("Importing {} pairs for cluster {}", size, f.getName()); + cs.runShell(cmd3, portInner); scan = jedis.scan("0", sc); size = scan.getResult().size(); } - log.info("Scanned {} for cluster {}",String.valueOf(size),f.getName()); + log.info("Scanned {} for cluster {}", String.valueOf(size), f.getName()); + + Pattern pattern = Pattern.compile(","); + String csvFile = pair.getPath(); + csvFile = csvFile.replace("txt","index"); + try (BufferedReader in = new BufferedReader(new FileReader(csvFile));) { + Map namefreq = in + .lines() + .map(x -> pattern.split(x)) + .collect(HashMap::new, (map, x) -> + map.put(f.getName()+"-"+pair.getName().split("\\.")[0]+"-"+x[0], x[1]), + Map::putAll); + + for (Map.Entry entry : namefreq.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + jedis.select(1); + jedis.set(key, value); + } - String clusterName = f.getName().replaceAll("[^0-9]+", ""); + + } + + +// String clusterName = f.getName().replaceAll("[^0-9]+", ""); //76 scan.getResult().parallelStream() - .forEach(m -> coreCompare(m, jedisPool, clusterName,outerPool,type)); + .forEach(m -> coreCompare(m, jedisPool, f.getName(), outerPool, type)); jedis.save(); @@ -108,6 +139,7 @@ public class MultiThreadTreeLoaderCluster { // } + } } String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; String stopServer1 = String.format(stopServer,Integer.valueOf(portInner)); @@ -122,26 +154,40 @@ public class MultiThreadTreeLoaderCluster { - public static Pair getTree(String firstValue, JedisPool outerPool,String type){ + public static Pair getTree(String prefix,String firstValue, JedisPool outerPool,JedisPool innerPool){ ITree tree = null; Jedis inner = null; - String[] split2 = firstValue.split("/"); - - String fullFileName = split2[split2.length-1]; - String[] split = fullFileName.split(".txt_"); - String pureFileName = split[0]; - String[] splitPJ = split[1].split("_"); - String project = splitPJ[1]; - String actionSetPosition = splitPJ[0]; + Jedis outer = null; +// String[] split2 = firstValue.split("/"); +// +// String fullFileName = split2[split2.length-1]; +// String[] split = fullFileName.split(".txt_"); +// String pureFileName = split[0]; +// String[] splitPJ = split[1].split("_"); +// String project = splitPJ[1]; +// String actionSetPosition = splitPJ[0]; try { - inner = outerPool.getResource(); - String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition; - String si= inner.get(filename); + inner = innerPool.getResource(); + inner.select(1); + String dist2load = inner.get(prefix+"-"+firstValue); + outer = outerPool.getResource(); + outer.select(0); + String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load); + Set keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load); + if(keys.size() == 1) { + s = (String) keys.toArray()[0]; + }else{ + throw new Error("cok key"); + } + + +// String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition; + String si= outer.get(s); HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si); @@ -161,8 +207,11 @@ public class MultiThreadTreeLoaderCluster { if (inner != null) { inner.close(); } + if (outer != null) { + outer.close(); + } } - Pair pair = new Pair<>(tree,project); + Pair pair = new Pair<>(tree,null); return pair; @@ -197,19 +246,20 @@ public class MultiThreadTreeLoaderCluster { catch (Exception e){ e.printStackTrace(); } - String firstValue = resultMap.get("0"); - String secondValue = resultMap.get("1"); +// String firstValue = resultMap.get("0"); +// String secondValue = resultMap.get("1"); try { - Pair oldPair = getTree(firstValue, outerPool,type); - Pair newPair = getTree(secondValue, outerPool,type); + String keyName = split[0]; + Pair oldPair = getTree(keyName,i, outerPool,jedisPool); + Pair newPair = getTree(keyName,j, outerPool,jedisPool); ITree oldTree = oldPair.getValue0(); ITree newTree = newPair.getValue0(); - String oldProject = oldPair.getValue1(); - String newProject = newPair.getValue1(); +// String oldProject = oldPair.getValue1(); +// String newProject = newPair.getValue1(); @@ -231,23 +281,24 @@ public class MultiThreadTreeLoaderCluster { String editDistance = String.valueOf(actions.size()); // jedis.select(1); - String result = resultMap.get("0") + "," + oldProject +"," + resultMap.get("1") + "," +newProject+ "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; + String result = i+ "," + j + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; // jedis.set(resultKey, result); + String matchKey = keyName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { - String matchKey = "match-"+clusterName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); - jedis.select(1); + jedis.select(2); jedis.set(matchKey, result); } jedis.select(0); - String pairKey = "pair_" + (String.valueOf(i)) + "_" + String.valueOf(j); - jedis.del(pairKey); + + jedis.del(matchKey); // log.info("Completed " + resultKey); }catch (Exception e){ log.debug(e.toString() + " {}",(name)); + e.printStackTrace(); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java index 2fbb676..e04f260 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java @@ -38,42 +38,55 @@ public class StoreEDiffInCache { File folder = new File(inputPath); File[] subFolders = folder.listFiles(); Stream stream = Arrays.stream(subFolders); - List pjs = stream + List roots = stream .filter(x -> !x.getName().startsWith(".")) .collect(Collectors.toList()); List workList = new ArrayList(); File[] dumps; - for (File pj : pjs) { - String pjName = pj.getName(); - File[] files = pj.listFiles(); - Stream fileStream = Arrays.stream(files); - List fs; -// if (operation.equals("ALLOP")){ -// fs= fileStream -// .filter(x -> x.getName().startsWith("UPD") || -// x.getName().startsWith("INS") || -// x.getName().startsWith("DEL") || -// x.getName().startsWith("MOV")) -// .collect(Collectors.toList()); -// File[] files1 = fs.get(0).listFiles(); -// File[] files2 = fs.get(1).listFiles(); -// File[] files3 = fs.get(2).listFiles(); -// File[] files4 = fs.get(3).listFiles(); -// dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new); -// }else{ - fs = fileStream - .filter(x -> x.getName().startsWith(operation)) - .collect(Collectors.toList()); - dumps = fs.get(0).listFiles(); -// } + for (File root : roots) { + String pjName = root.getName(); + File[] sizes = root.listFiles(); + Stream sizesStream = Arrays.stream(sizes); + List sizeFolders = sizesStream + .filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); + + for (File sf:sizeFolders){ + File[] files = sf.listFiles(); + int length = files.length; + if (length == 1) + continue; + + Stream fileStream = Arrays.stream(files); + List fs; + // if (operation.equals("ALLOP")){ + // fs= fileStream + // .filter(x -> x.getName().startsWith("UPD") || + // x.getName().startsWith("INS") || + // x.getName().startsWith("DEL") || + // x.getName().startsWith("MOV")) + // .collect(Collectors.toList()); + // File[] files1 = fs.get(0).listFiles(); + // File[] files2 = fs.get(1).listFiles(); + // File[] files3 = fs.get(2).listFiles(); + // File[] files4 = fs.get(3).listFiles(); + // dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new); + // }else{ +// fs = fileStream +// .filter(x -> x.getName().startsWith(operation)) +// .collect(Collectors.toList()); +// dumps = fs.get(0).listFiles(); + // } - for (File f : dumps) { - String name = f.getName(); + for (File f : files) { + String name = f.getName(); + + String key = pjName + "/"+ sf.getName()+"/" + name; + String result = key +","+f.getPath(); + workList.add(result); + } - String key = pjName + "/"+ operation+"/" + name; - String result = key +","+f.getPath(); - workList.add(result); } } diff --git a/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java b/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java new file mode 100644 index 0000000..0285940 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java @@ -0,0 +1,71 @@ +package edu.lu.uni.serval.utils; + +import com.github.gumtreediff.tree.ITree; +import com.github.gumtreediff.tree.TreeContext; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class ClusterToPattern { + + public static void main(String[] args) throws IOException, ClassNotFoundException { + String shapesFolder = "/Users/anil.koyuncu/projects/fixminer-all/enhancedASTDiff/python/data/shapes"; + + String port = "6399"; + JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(port),20000000); + + File folder = new File(shapesFolder); + + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List shapes = stream + .filter(x -> !x.getName().startsWith(".")) + .filter(x -> !x.getName().endsWith(".pickle")) + .collect(Collectors.toList()); + for(File shape:shapes){ + File[] clusters = shape.listFiles(); + Stream fileStream = Arrays.stream(clusters); + List fileList = fileStream.filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); + for(File cluster:fileList){ + File[] files = cluster.listFiles(); + File file = files[0]; + String key = shape.getName() + "/*/"+file.getName(); + Jedis outer = outerPool.getResource(); + String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load); + Set keys = outer.keys(key); + if (keys.size() == 1) { + s = (String) keys.toArray()[0]; + } else { + throw new Error("cok key"); + } + + String si = outer.get(s); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si); + + + String s1 = actionSet.toString(); + File saveFile = new File(folder.getParent()+"/shapePatterns/" +shape.getName()+"/"+cluster.getName()+".pattern"); + saveFile.getParentFile().mkdirs(); + BufferedWriter writer = new BufferedWriter(new FileWriter(saveFile)); + writer.write(s1); + + writer.close(); + + + } + + } + + } +} diff --git a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java index 10fddfe..766f154 100644 --- a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java +++ b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java @@ -10,10 +10,7 @@ import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import java.io.*; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; /** @@ -130,7 +127,7 @@ public class EDiffHelper { .collect(Collectors.toList()); } - public static ITree getSimpliedTree(String fn, JedisPool outerPool) { + public static ITree getSimpliedTree(String prefix, String fn, JedisPool outerPool) { ITree tree = null; Jedis inner = null; @@ -140,9 +137,9 @@ public class EDiffHelper { log.info("wait"); } inner.select(1); - String dist2load = inner.get(fn); + String dist2load = inner.get(prefix+"-"+fn); inner.select(0); - String s = inner.get(dist2load); + String s = inner.get(prefix.replace("-","/") +"/"+dist2load); HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s); ITree parent = null; @@ -164,6 +161,152 @@ public class EDiffHelper { return tree; } + + + public static ITree getShapes(String prefix, String fn, JedisPool outerPool) { + + ITree tree = null; + Jedis inner = null; + try { + inner = outerPool.getResource(); + while (!inner.ping().equals("PONG")){ + log.info("wait"); + } + inner.select(1); + String dist2load = inner.get(prefix+"-"+fn); + inner.select(0); + String s = inner.get(prefix.replace("-","/") +"/"+dist2load); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s); + + ITree parent = null; + ITree children =null; + TreeContext tc = new TreeContext(); + tree = EDiffHelper.getASTTree(actionSet, parent, children,tc); + //tree.setParent(null); + tc.validate(); + + } catch (IOException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + }finally { + if (inner != null) { + inner.close(); + } + } + return tree; + + } + + + public static ITree getActions(String prefix,String firstValue, JedisPool outerPool,JedisPool innerPool) { + + + ITree tree = null; + Jedis inner = null; + Jedis outer = null; + + + try { + inner = innerPool.getResource(); + inner.select(1); + String dist2load = inner.get(prefix + "-" + firstValue); + outer = outerPool.getResource(); + outer.select(0); + String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load); + Set keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load); + if (keys.size() == 1) { + s = (String) keys.toArray()[0]; + } else { + throw new Error("cok key"); + } + + +// String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition; + String si = outer.get(s); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si); + + + ITree parent = null; + ITree children = null; + TreeContext tc = new TreeContext(); + tree = EDiffHelper.getActionTree(actionSet, parent, children, tc); + //tree.setParent(null); + tc.validate(); + + + } catch (IOException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } finally { + if (inner != null) { + inner.close(); + } + if (outer != null) { + outer.close(); + } + } + + return tree; + } + + + public static ITree getTokens(String prefix,String firstValue, JedisPool outerPool,JedisPool innerPool) { + + + ITree tree = null; + Jedis inner = null; + Jedis outer = null; + + + try { + inner = innerPool.getResource(); + inner.select(1); + String dist2load = inner.get(prefix + "-" + firstValue); + outer = outerPool.getResource(); + outer.select(0); + String s = null;//inner.get(prefix.replace("-","/") +"/"+dist2load); + Set keys = outer.keys(prefix.split("-")[0] + "/*/" + dist2load); + if (keys.size() == 1) { + s = (String) keys.toArray()[0]; + } else { + throw new Error("cok key"); + } + + +// String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition; + String si = outer.get(s); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(si); + + + ITree parent = null; + ITree children = null; + TreeContext tc = new TreeContext(); + tree = EDiffHelper.getTokenTree(actionSet, parent, children, tc); + tree.setParent(null); + tc.validate(); + + + } catch (IOException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } finally { + if (inner != null) { + inner.close(); + } + if (outer != null) { + outer.close(); + } + } + + return tree; + } + + + + public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){ int newType = 0;