diff --git a/pom.xml b/pom.xml
index 174b076..ebaa43d 100755
--- a/pom.xml
+++ b/pom.xml
@@ -102,11 +102,11 @@
1.3
-
- com.rabbitmq
- amqp-client
- 4.0.0
-
+
+
+
+
+
junit
junit
@@ -114,17 +114,17 @@
-
- org.apache.lucene
- lucene-core
- 4.8.0
-
+
+
+
+
+
-
- org.apache.lucene
- lucene-analyzers-common
- 4.8.0
-
+
+
+
+
+
org.apache.commons
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
index a0d0456..41160e0 100755
--- a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
@@ -22,7 +22,7 @@ public class Launcher {
Properties appProps = new Properties();
-// String appConfigPath = "/Users/haoyetian/Documents/Lu_code/FixMiner/fixminer_source/src/main/resource/app.properties";
+// String appConfigPath = "/Users/anil.koyuncu/projects/test/fixminerC/src/main/resource/app.properties";
String appConfigPath = args[0];
appProps.load(new FileInputStream(appConfigPath));
@@ -40,8 +40,12 @@ public class Launcher {
String parameter = args[2];
// String parameter = null;
+// String parameter = "decl";
+// String parameter = "L2";
String jobType = args[1];
// String jobType = "RICHEDITSCRIPT";
+// String jobType = "LOAD";
+// String jobType = "COMPARE";
// String parameters = String.format("\nportInner %s " +
// "\nnumOfWorkers %s " +
@@ -73,7 +77,11 @@ public class Launcher {
try {
switch (jobType) {
case "RICHEDITSCRIPT":
- EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath);
+ EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter);
+ break;
+
+ case "LOAD":
+ EnhancedASTDiff.load(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter);
break;
case "COMPARE":
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java
index d00ffbb..345ca70 100755
--- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java
@@ -7,6 +7,7 @@ import redis.clients.jedis.JedisPool;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -71,9 +72,38 @@ public class AkkaTreeParser {
}
- log.info("Getting results");
+// log.info("Getting results %d",fileMap.s);
return fileMap;
}
+ public static List files2compare(JedisPool innerPool){
+
+
+// HashMap fileMap =new HashMap();
+ List result = new ArrayList();
+ try (Jedis inner = innerPool.getResource()) {
+ while (!inner.ping().equals("PONG")){
+ log.info("wait");
+ }
+
+// inner.select(1);
+ Map filenames = inner.hgetAll("compare");
+
+
+ for (Map.Entry stringStringEntry : filenames.entrySet().stream().collect(Collectors.toList())) {
+// fileMap.put(stringStringEntry.getKey(),stringStringEntry.getValue());
+ result.add(stringStringEntry.getKey());
+ }
+
+
+
+
+
+ }
+
+ log.info("Getting results :" + result.size());
+ return result;
+ }
+
}
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java
index 3d579e0..0b67c31 100755
--- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java
@@ -37,21 +37,22 @@ public class CompareTrees {
log.info(cmd);
cs.runShell(cmd, port);
- String cmdInner = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s";
- cmdInner = String.format(cmdInner, redisPath,compareDBName,Integer.valueOf(portInner));
- log.info(cmdInner);
- cs.runShell(cmdInner, portInner);
+// String cmdInner = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s";
+// cmdInner = String.format(cmdInner, redisPath,compareDBName,Integer.valueOf(portInner));
+// log.info(cmdInner);
+// cs.runShell(cmdInner, portInner);
String numOfWorkers = "100000000";//args[4];
String host = "localhost";//args[5];
// -Djava.util.concurrent.ForkJoinPool.common.parallelism=256
- final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000);
+// final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000);
final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(port),20000000);
- List listOfPairs = AkkaTreeParser.getMessages(innerPool,Integer.valueOf(numOfWorkers));
- HashMap filenames = AkkaTreeParser.filenames(innerPool);
+// List listOfPairs = AkkaTreeParser.getMessages(innerPool,Integer.valueOf(numOfWorkers));
+ HashMap filenames = AkkaTreeParser.filenames(outerPool);
+ List listOfPairs = AkkaTreeParser.files2compare(outerPool);
ArrayList samePairs = new ArrayList<>();
@@ -59,14 +60,15 @@ public class CompareTrees {
- listOfPairs.stream().parallel().forEach(m->coreCompare(m, job,innerPool, samePairs,errorPairs,filenames,outerPool));
+ listOfPairs.stream().parallel().forEach(m->coreCompare(m, job,null, samePairs,errorPairs,filenames,outerPool));
- try (Jedis jedis = innerPool.getResource()) {
+ try (Jedis jedis = outerPool.getResource()) {
jedis.select(0);
- jedis.flushDB();
+// jedis.flushDB();
+ jedis.del("compare");
for (String errorPair : errorPairs) {
- jedis.hset(errorPair, "0", "1");
+ jedis.hset("compare", errorPair, "1");
}
@@ -85,6 +87,8 @@ public class CompareTrees {
Pair newPair = null;
String matchKey = null;
+ innerPool = outerPool;
+
try {
String[] split = pairName.split("/");
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/DefaultKryoContext.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/DefaultKryoContext.java
index 7941ab1..3a71d4a 100644
--- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/DefaultKryoContext.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/DefaultKryoContext.java
@@ -5,6 +5,8 @@ import java.io.ByteArrayOutputStream;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
+import com.esotericsoftware.kryo.io.UnsafeInput;
+import com.esotericsoftware.kryo.io.UnsafeOutput;
import com.esotericsoftware.kryo.pool.KryoFactory;
import com.esotericsoftware.kryo.pool.KryoPool;
import org.objenesis.strategy.SerializingInstantiatorStrategy;
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java
index 5316137..425643a 100755
--- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java
@@ -54,7 +54,7 @@ public class EDiffActor extends UntypedActor {
int toIndex = (i + 1) * average + counter;
List filesOfWorkers = files.subList(fromIndex, toIndex);
- final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getInnerPool(),((EDiffMessage) message).getSrcMLPath());
+ final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getInnerPool(),((EDiffMessage) message).getSrcMLPath(),((EDiffMessage) message).getRootType());
mineRouter.tell(workMsg, getSelf());
logger.info("Assign {} task to worker #" + (i + 1) ,filesOfWorkers.size());
}
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java
index d6bcdd6..81d8fe0 100755
--- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java
@@ -1,6 +1,7 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
import edu.lu.uni.serval.utils.EDiffHelper;
+import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
@@ -24,7 +25,7 @@ public class EDiffHunkParser extends EDiffParser {
private static Logger logger = LoggerFactory.getLogger(EDiffHunkParser.class);
@Override
- public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath) {
+ public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath,String rootType) {
List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile,srcMLPath);
if (actionSets.size() != 0) {
@@ -38,6 +39,9 @@ public class EDiffHunkParser extends EDiffParser {
try {
String astNodeType = actionSet.getAstNodeType();
+ if (astNodeType.equals(rootType)){
+
+ }
actionSet.toString();
int size = actionSet.getActionSize();
@@ -49,15 +53,15 @@ public class EDiffHunkParser extends EDiffParser {
String key = astNodeType+"/"+String.valueOf(size)+"/" + pj +"_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
- try (Jedis inner = innerPool.getResource()) {
+// try (Jedis inner = innerPool.getResource()) {
+//
+// inner.hset("dump".getBytes(),key.getBytes(),EDiffHelper.kryoSerialize(actionSet));
+// }
+ File f = new File(root+"dumps/"+astNodeType+"/"+String.valueOf(size)+"/");
+ f.mkdirs();
+ f = new File(root+"dumps/"+key);
- inner.hset("dump".getBytes(),key.getBytes(),EDiffHelper.kryoSerialize(actionSet));
- }
-// File f = new File(root+"dumps/"+astNodeType+"/"+String.valueOf(size)+"/");
-// f.mkdirs();
-// f = new File(root+"dumps/"+key);
-//
-//
+ FileUtils.writeByteArrayToFile(f,EDiffHelper.kryoSerialize(actionSet));
// FileOutputStream fos = new FileOutputStream(f);
// ObjectOutputStream oos = new ObjectOutputStream(fos);
// oos.writeObject(EDiffHelper.kryoSerialize(actionSet));
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java
index b70960c..e8bbbb0 100755
--- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java
@@ -27,15 +27,22 @@ public class EDiffMessage extends BaseMessage{
private String srcMLPath;
+ public String getRootType() {
+ return rootType;
+ }
+
+ private String rootType;
+
- public EDiffMessage(int id, List msgFiles,String eDiffTimeout,JedisPool pool,String srcMLPath) {
+ public EDiffMessage(int id, List msgFiles,String eDiffTimeout,JedisPool pool,String srcMLPath,String rootType) {
super(id,new Long(eDiffTimeout));
this.msgFiles = msgFiles;
this.innerPool = pool;
this.srcMLPath = srcMLPath;
+ this.rootType = rootType;
}
public EDiffMessage(int id, List msgFiles,Long eDiffTimeout,JedisPool pool) {
super(id,eDiffTimeout);
@@ -44,12 +51,13 @@ public class EDiffMessage extends BaseMessage{
}
- public EDiffMessage(int id, List filesOfWorkers, long seconds_to_wait, JedisPool innerPool, String srcMLPath) {
+ public EDiffMessage(int id, List filesOfWorkers, long seconds_to_wait, JedisPool innerPool, String srcMLPath,String rootType) {
super(id,seconds_to_wait);
this.msgFiles = filesOfWorkers;
this.innerPool = innerPool;
this.srcMLPath = srcMLPath;
+ this.rootType = rootType;
}
public List getMsgFiles() {
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java
index 2d2cf35..4856b1f 100755
--- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java
@@ -81,7 +81,7 @@ public class EDiffParser extends Parser {
}
@Override
- public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool, String srcMLPath) {
+ public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool, String srcMLPath,String rootType) {
}
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java
index 960a770..b1ee053 100755
--- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java
@@ -44,6 +44,7 @@ public class EDiffWorker extends UntypedActor {
int counter = 0;
JedisPool innerPool = msg.getInnerPool();
String srcMLPath = msg.getSrcMLPath();
+ String rootType = msg.getRootType();
for (MessageFile msgFile : files) {
File revFile = msgFile.getRevFile();
@@ -58,7 +59,7 @@ public class EDiffWorker extends UntypedActor {
// schedule the work
- final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getInnerPool(),srcMLPath));
+ final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getInnerPool(),srcMLPath,rootType));
try {
// wait for task to complete
future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS);
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java
index ffdb4c6..1df607a 100644
--- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java
@@ -1,10 +1,18 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser;
+import edu.lu.uni.serval.utils.EDiffHelper;
+import edu.lu.uni.serval.utils.PoolBuilder;
+import org.apache.commons.io.FileUtils;
import org.junit.Test;
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
import java.io.File;
import java.io.IOException;
+import java.lang.instrument.Instrumentation;
+import java.time.Duration;
+import java.time.Instant;
public class HunkParserTest {
@@ -12,12 +20,20 @@ public class HunkParserTest {
public void testSimple() throws IOException {
// String input = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/revFiles/7f52f3_3845d29_drivers#pci#host#pcie-altera.c";
- String root = "/Users/anilkoyuncu/projects/gumInputLinux/linux/";
+ String root = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/linux/";
-// String filename = "8dd302_c4ef85_net#core#dev_ioctl.c";
- String filename = "6a28fd_93ad867_drivers#tty#goldfish.c";
+// String filename = "8dd302_c4ef85_net#core#dev_ioctl.c"; //ok
+// String filename = "1e793f6_77f18a_drivers#scsi#megaraid#megaraid_sas_base.c"; //OK
+// String filename = "6a28fd_93ad867_drivers#tty#goldfish.c"; //m,issing
+// String filename = "b90f7c_ff51ff_kernel#sched#fair.c"; //wrong and wired
+// String filename = "ed8f68_b1c8047_fs#ext3#dir.c";//ok
+
+// String filename = "bc3d12_9a26653_drivers#scsi#libfc#fc_disc.c"; //okish
+ String filename = "118154_0c5f81_arch#x86#kvm#svm.c";
+// String filename = "bcbd94f_43e43c9_drivers#md#dm-crypt.c"; //emin degilim
+// String filename = "f1727b4_6c1e7e_arch#x86#kvm#vmx#nested.c"; //komplex not sure
// String filename = "5924f17_5925a05_net#ipv4#tcp.c";
-// String filename = "bd0b9ac_b237721_drivers#irqchip#irq-dw-apb-ictl.c";
+// String filename = "bd0b9ac_b237721_drivers#irqchip#irq-dw-apb-ictl.c"; //missing
// String filename = "052831_3985e8_include#net#ip_tunnels.h";
// String filename = "e76019_28647b_drivers#gpu#drm#i915#i915_drv.h";
// String filename = "4cbe4d_b124f4_include#linux#mlx4#device.h"; //enum case stops at block
@@ -26,10 +42,11 @@ public class HunkParserTest {
File prevFile =new File(root + "prevFiles/prev_"+filename);
EDiffHunkParser parser = new EDiffHunkParser();
- String srcMLPath = "/Users/anilkoyuncu/Downloads/srcML2/src2srcml";
+ String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML/src2srcml";
parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath);
// ITree t = new SrcmlCppTreeGenerator().generateFromFile(input).getRoot();
// Assert.assertEquals(148, t.getSize());
+
}
@Test
@@ -42,7 +59,7 @@ public class HunkParserTest {
File prevFile =new File(root + "prevFiles/prev_"+filename);
File diffFile = new File(root + "DiffEntries/"+filename+".txt");
EDiffHunkParser parser = new EDiffHunkParser();
- parser.parseFixPatterns(prevFile,revFile,diffFile,"gumInputLinux",null,srcMLPath);
+ parser.parseFixPatterns(prevFile,revFile,diffFile,"gumInputLinux",null,srcMLPath,null);
}
@@ -64,4 +81,46 @@ public class HunkParserTest {
// Assert.assertEquals(148, t.getSize());
}
+
+
+ @Test
+ public void rich() throws IOException {
+ Instant start = Instant.now();
+ final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "localhost",Integer.valueOf("6399"),20000000);
+
+ EDiffHunkParser parser = new EDiffHunkParser();
+ String root = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/linux/";
+ String filename = "bb67dd_0922c7_sound#soc#sof#intel#hda.c";
+ File revFile = new File(root + "revFiles/"+ filename);
+ File prevFile =new File(root + "prevFiles/prev_"+filename);
+ File diffFile = new File(root + "DiffEntries/"+filename+".txt");
+ String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML/src2srcml";
+ parser.parseFixPatterns(prevFile,revFile,diffFile,"gumInputLinux",outerPool,srcMLPath,"if");
+ String key = "if/3/linux_bb67dd_0922c7_sound#soc#sof#intel#hda.c.txt_0";
+ File file2load = new File("/Users/anil.koyuncu/projects/test/fixminer-core/python/data/dumps/"+ key);
+ byte[] dump = FileUtils.readFileToByteArray(file2load);
+
+ HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(dump);
+ actionSet.toString();
+
+
+// CODE HERE
+ Instant finish = Instant.now();
+ long timeElapsed = Duration.between(start, finish).toMillis();
+ System.out.println(timeElapsed);
+
+// try (Jedis outer = outerPool.getResource()) {
+// try {
+//
+//
+//
+// byte[] s = outer.hget("dump".getBytes(), key.getBytes());
+// HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(s);
+// actionSet.getActionSize();
+// } catch (Exception e) {
+// e.printStackTrace();
+// }
+// }
+ }
+
}
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java
index a738898..91c68e6 100755
--- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java
@@ -20,7 +20,7 @@ public abstract class Parser implements ParserInterface {
protected String originalTree = ""; // Guide of generating patches.
protected String actionSets = ""; // Guide of generating patches.
- public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool, String srcMLPath);
+ public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool, String srcMLPath,String rootType);
@Override
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java
index 4c23d50..d0ee8ab 100755
--- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java
@@ -13,7 +13,8 @@ public class RunnableParser implements Runnable {
private String project;
private JedisPool pool;
private String srcMLPath;
-
+ private String rootType;
+
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) {
this.prevFile = prevFile;
this.revFile = revFile;
@@ -30,7 +31,7 @@ public class RunnableParser implements Runnable {
this.pool = innerPool;
}
- public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool,String srcMLPath) {
+ public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool,String srcMLPath,String rootType) {
this.prevFile = prevFile;
this.revFile = revFile;
this.diffentryFile = diffentryFile;
@@ -38,10 +39,11 @@ public class RunnableParser implements Runnable {
this.project = project;
this.pool = innerPool;
this.srcMLPath = srcMLPath;
+ this.rootType = rootType;
}
@Override
public void run() {
- parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,pool,srcMLPath);
+ parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,pool,srcMLPath,rootType);
}
}
diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
index 7ed5b90..d96a6c7 100755
--- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
+++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
@@ -2,18 +2,20 @@ package edu.lu.uni.serval.fixminer.jobs;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
-import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor;
-import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser;
-import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage;
-import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile;
+import edu.lu.uni.serval.fixminer.akka.ediff.*;
import edu.lu.uni.serval.utils.CallShell;
+import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
+import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
-import java.io.File;
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -24,7 +26,7 @@ public class EnhancedASTDiff {
private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class);
- public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath) throws Exception {
+ public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType) throws Exception {
String parameters = String.format("\nInput path %s",inputPath);
@@ -32,7 +34,11 @@ public class EnhancedASTDiff {
CallShell cs = new CallShell();
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
- cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
+ if (rootType == null){
+ cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
+ }else{
+ cmd = String.format(cmd, dbDir,rootType+chunkName,Integer.valueOf(portInner));
+ }
cs.runShell(cmd, portInner);
@@ -64,7 +70,7 @@ public class EnhancedASTDiff {
ActorSystem system = null;
ActorRef parsingActor = null;
- final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath);
+ final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath,rootType);
try {
log.info("Akka begins...");
log.info("{} files to process ...", allMessageFiles.size());
@@ -88,7 +94,7 @@ public class EnhancedASTDiff {
forEach(m ->
{
EDiffHunkParser parser = new EDiffHunkParser();
- parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool,"/Users/anilkoyuncu/Downloads/srcML2/src2srcml");
+ parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool,"/Users/anilkoyuncu/Downloads/srcML2/src2srcml",null);
if (counter % 10 == 0) {
log.info("Finalized parsing " + counter + " files... remaining " + (allMessageFiles.size() - counter));
}
@@ -141,5 +147,69 @@ public class EnhancedASTDiff {
}
}
+ public static void load(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType) throws Exception {
+
+
+ String parameters = String.format("\nInput path %s",inputPath);
+ log.info(parameters);
+
+ CallShell cs = new CallShell();
+ String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
+ cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
+// if (rootType == null){
+// cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
+// }else{
+// cmd = String.format(cmd, dbDir,rootType+chunkName,Integer.valueOf(portInner));
+// }
+
+ cs.runShell(cmd, portInner);
+
+ JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000);
+ try (Jedis inner = innerPool.getResource()) {
+ inner.flushAll();
+ }
+ File folder = new File(new File(inputPath).getParent() + "/dumps/" + rootType);
+ File[] listOfFiles = folder.listFiles();
+ Stream stream = Arrays.stream(listOfFiles);
+ List folders = stream
+ .filter(x -> !x.getName().startsWith("."))
+ .collect(Collectors.toList());
+ List allMessageFiles = new ArrayList<>();
+
+ for (File target : folders) {
+ if(target.getName().startsWith("."))
+ continue;
+ List files = Arrays.asList(target.listFiles());
+ if (files.size() > 1){
+ allMessageFiles.addAll(files);
+ }
+ }
+
+ log.info("Message size: "+allMessageFiles.size());
+
+ allMessageFiles.stream().
+ parallel().
+ forEach(x-> loadCore(x,innerPool));
+
+ }
+
+ public static void loadCore(File file2load, JedisPool innerPool){
+ try (Jedis inner = innerPool.getResource()) {
+
+// byte[] dump = Files.readAllBytes(Paths.get(file2load.getPath()));
+ byte[] dump = FileUtils.readFileToByteArray(file2load);
+// HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(dump);
+ String key = file2load.getPath().split("/dumps/")[1];
+ inner.hset("dump".getBytes(),key.getBytes(),dump);
+// actionSet.toString();
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+
}
diff --git a/src/main/resource/app.properties b/src/main/resource/app.properties
index 62363bb..a6ab30f 100755
--- a/src/main/resource/app.properties
+++ b/src/main/resource/app.properties
@@ -3,11 +3,11 @@ pjName = gumInput
portInner = 6380
portDumps = 6399
parallelism = AKKA
-numOfWorkers = 1
+numOfWorkers = 14
-inputPath = /Users/anilkoyuncu/projects/gumInputLinux/
-redisPath = /Users/anilkoyuncu/projects/fixminer/fixminer-core/python/data/redis
-srcMLPath= /Users/anilkoyuncu/Downloads/srcML2/src2srcml
+inputPath = /Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux
+redisPath = /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis
+srcMLPath= /Users/anil.koyuncu/Downloads/srcML/src2srcml
actionType =ALL
eDiffTimeout = 900