azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changes for single rootType

Browse Source
This commit is contained in:
mimic
2020-01-09 14:11:27 +01:00
parent 422a3546b6
commit 49a0aeba0b
15 changed files with 254 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pom.xml
+15 -15
View File
@@ -102,11 +102,11 @@
<version>1.3</version>
</dependency>
<dependency>
<groupId>com.rabbitmq</groupId>
<artifactId>amqp-client</artifactId>
<version>4.0.0</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.rabbitmq</groupId>-->
<!-- <artifactId>amqp-client</artifactId>-->
<!-- <version>4.0.0</version>-->
<!-- </dependency>-->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
@@ -114,17 +114,17 @@
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.lucene/lucene-core -->
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>4.8.0</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.apache.lucene</groupId>-->
<!-- <artifactId>lucene-core</artifactId>-->
<!-- <version>4.8.0</version>-->
<!-- </dependency>-->
<!-- https://mvnrepository.com/artifact/org.apache.lucene/lucene-analyzers-common -->
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>4.8.0</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.apache.lucene</groupId>-->
<!-- <artifactId>lucene-analyzers-common</artifactId>-->
<!-- <version>4.8.0</version>-->
<!-- </dependency>-->
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-math3 -->
<dependency>
<groupId>org.apache.commons</groupId>
src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+10 -2
View File
@@ -22,7 +22,7 @@ public class Launcher {
Properties appProps = new Properties();
// String appConfigPath = "/Users/haoyetian/Documents/Lu_code/FixMiner/fixminer_source/src/main/resource/app.properties";
// String appConfigPath = "/Users/anil.koyuncu/projects/test/fixminerC/src/main/resource/app.properties";
String appConfigPath = args[0];
appProps.load(new FileInputStream(appConfigPath));
@@ -40,8 +40,12 @@ public class Launcher {
String parameter = args[2];
// String parameter = null;
// String parameter = "decl";
// String parameter = "L2";
String jobType = args[1];
// String jobType = "RICHEDITSCRIPT";
// String jobType = "LOAD";
// String jobType = "COMPARE";
// String parameters = String.format("\nportInner %s " +
// "\nnumOfWorkers %s " +
@@ -73,7 +77,11 @@ public class Launcher {
try {
switch (jobType) {
case "RICHEDITSCRIPT":
EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath);
EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter);
break;
case "LOAD":
EnhancedASTDiff.load(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter);
break;
case "COMPARE":
src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java
+31 -1
View File
@@ -7,6 +7,7 @@ import redis.clients.jedis.JedisPool;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -71,9 +72,38 @@ public class AkkaTreeParser {
}
log.info("Getting results");
// log.info("Getting results %d",fileMap.s);
return fileMap;
}
public static List<String> files2compare(JedisPool innerPool){
// HashMap<String, String> fileMap =new HashMap<String, String>();
List<String> result = new ArrayList<String>();
try (Jedis inner = innerPool.getResource()) {
while (!inner.ping().equals("PONG")){
log.info("wait");
}
// inner.select(1);
Map<String, String> filenames = inner.hgetAll("compare");
for (Map.Entry<String, String> stringStringEntry : filenames.entrySet().stream().collect(Collectors.toList())) {
// fileMap.put(stringStringEntry.getKey(),stringStringEntry.getValue());
result.add(stringStringEntry.getKey());
}
}
log.info("Getting results :" + result.size());
return result;
}
}
src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java
+15 -11
View File
@@ -37,21 +37,22 @@ public class CompareTrees {
log.info(cmd);
cs.runShell(cmd, port);
String cmdInner = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s";
cmdInner = String.format(cmdInner, redisPath,compareDBName,Integer.valueOf(portInner));
log.info(cmdInner);
cs.runShell(cmdInner, portInner);
// String cmdInner = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s";
// cmdInner = String.format(cmdInner, redisPath,compareDBName,Integer.valueOf(portInner));
// log.info(cmdInner);
// cs.runShell(cmdInner, portInner);
String numOfWorkers = "100000000";//args[4];
String host = "localhost";//args[5];
// -Djava.util.concurrent.ForkJoinPool.common.parallelism=256
final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000);
// final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000);
final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(port),20000000);
List<String> listOfPairs = AkkaTreeParser.getMessages(innerPool,Integer.valueOf(numOfWorkers));
HashMap<String, String> filenames = AkkaTreeParser.filenames(innerPool);
// List<String> listOfPairs = AkkaTreeParser.getMessages(innerPool,Integer.valueOf(numOfWorkers));
HashMap<String, String> filenames = AkkaTreeParser.filenames(outerPool);
List<String> listOfPairs = AkkaTreeParser.files2compare(outerPool);
ArrayList<String> samePairs = new ArrayList<>();
@@ -59,14 +60,15 @@ public class CompareTrees {
listOfPairs.stream().parallel().forEach(m->coreCompare(m, job,innerPool, samePairs,errorPairs,filenames,outerPool));
listOfPairs.stream().parallel().forEach(m->coreCompare(m, job,null, samePairs,errorPairs,filenames,outerPool));
try (Jedis jedis = innerPool.getResource()) {
try (Jedis jedis = outerPool.getResource()) {
jedis.select(0);
jedis.flushDB();
// jedis.flushDB();
jedis.del("compare");
for (String errorPair : errorPairs) {
jedis.hset(errorPair, "0", "1");
jedis.hset("compare", errorPair, "1");
}
@@ -85,6 +87,8 @@ public class CompareTrees {
Pair<ITree, HierarchicalActionSet> newPair = null;
String matchKey = null;
innerPool = outerPool;
try {
String[] split = pairName.split("/");
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/DefaultKryoContext.java
+2
View File
@@ -5,6 +5,8 @@ import java.io.ByteArrayOutputStream;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.esotericsoftware.kryo.io.UnsafeInput;
import com.esotericsoftware.kryo.io.UnsafeOutput;
import com.esotericsoftware.kryo.pool.KryoFactory;
import com.esotericsoftware.kryo.pool.KryoPool;
import org.objenesis.strategy.SerializingInstantiatorStrategy;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java
+1 -1
View File
@@ -54,7 +54,7 @@ public class EDiffActor extends UntypedActor {
int toIndex = (i + 1) * average + counter;
List<MessageFile> filesOfWorkers = files.subList(fromIndex, toIndex);
final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getInnerPool(),((EDiffMessage) message).getSrcMLPath());
final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getInnerPool(),((EDiffMessage) message).getSrcMLPath(),((EDiffMessage) message).getRootType());
mineRouter.tell(workMsg, getSelf());
logger.info("Assign {} task to worker #" + (i + 1) ,filesOfWorkers.size());
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java
+13 -9
View File
@@ -1,6 +1,7 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
import edu.lu.uni.serval.utils.EDiffHelper;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
@@ -24,7 +25,7 @@ public class EDiffHunkParser extends EDiffParser {
private static Logger logger = LoggerFactory.getLogger(EDiffHunkParser.class);
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath) {
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath,String rootType) {
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile,srcMLPath);
if (actionSets.size() != 0) {
@@ -38,6 +39,9 @@ public class EDiffHunkParser extends EDiffParser {
try {
String astNodeType = actionSet.getAstNodeType();
if (astNodeType.equals(rootType)){
}
actionSet.toString();
int size = actionSet.getActionSize();
@@ -49,15 +53,15 @@ public class EDiffHunkParser extends EDiffParser {
String key = astNodeType+"/"+String.valueOf(size)+"/" + pj +"_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
try (Jedis inner = innerPool.getResource()) {
// try (Jedis inner = innerPool.getResource()) {
//
// inner.hset("dump".getBytes(),key.getBytes(),EDiffHelper.kryoSerialize(actionSet));
// }
File f = new File(root+"dumps/"+astNodeType+"/"+String.valueOf(size)+"/");
f.mkdirs();
f = new File(root+"dumps/"+key);
inner.hset("dump".getBytes(),key.getBytes(),EDiffHelper.kryoSerialize(actionSet));
}
// File f = new File(root+"dumps/"+astNodeType+"/"+String.valueOf(size)+"/");
// f.mkdirs();
// f = new File(root+"dumps/"+key);
//
//
FileUtils.writeByteArrayToFile(f,EDiffHelper.kryoSerialize(actionSet));
// FileOutputStream fos = new FileOutputStream(f);
// ObjectOutputStream oos = new ObjectOutputStream(fos);
// oos.writeObject(EDiffHelper.kryoSerialize(actionSet));
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java
+10 -2
View File
@@ -27,15 +27,22 @@ public class EDiffMessage extends BaseMessage{
private String srcMLPath;
public String getRootType() {
return rootType;
}
private String rootType;
public EDiffMessage(int id, List<MessageFile> msgFiles,String eDiffTimeout,JedisPool pool,String srcMLPath) {
public EDiffMessage(int id, List<MessageFile> msgFiles,String eDiffTimeout,JedisPool pool,String srcMLPath,String rootType) {
super(id,new Long(eDiffTimeout));
this.msgFiles = msgFiles;
this.innerPool = pool;
this.srcMLPath = srcMLPath;
this.rootType = rootType;
}
public EDiffMessage(int id, List<MessageFile> msgFiles,Long eDiffTimeout,JedisPool pool) {
super(id,eDiffTimeout);
@@ -44,12 +51,13 @@ public class EDiffMessage extends BaseMessage{
}
public EDiffMessage(int id, List<MessageFile> filesOfWorkers, long seconds_to_wait, JedisPool innerPool, String srcMLPath) {
public EDiffMessage(int id, List<MessageFile> filesOfWorkers, long seconds_to_wait, JedisPool innerPool, String srcMLPath,String rootType) {
super(id,seconds_to_wait);
this.msgFiles = filesOfWorkers;
this.innerPool = innerPool;
this.srcMLPath = srcMLPath;
this.rootType = rootType;
}
public List<MessageFile> getMsgFiles() {
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java
+1 -1
View File
@@ -81,7 +81,7 @@ public class EDiffParser extends Parser {
}
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool, String srcMLPath) {
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool, String srcMLPath,String rootType) {
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java
+2 -1
View File
@@ -44,6 +44,7 @@ public class EDiffWorker extends UntypedActor {
int counter = 0;
JedisPool innerPool = msg.getInnerPool();
String srcMLPath = msg.getSrcMLPath();
String rootType = msg.getRootType();
for (MessageFile msgFile : files) {
File revFile = msgFile.getRevFile();
@@ -58,7 +59,7 @@ public class EDiffWorker extends UntypedActor {
// schedule the work
final Future<?> future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getInnerPool(),srcMLPath));
final Future<?> future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getInnerPool(),srcMLPath,rootType));
try {
// wait for task to complete
future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS);
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java
+65 -6
View File
@@ -1,10 +1,18 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.io.FileUtils;
import org.junit.Test;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.io.File;
import java.io.IOException;
import java.lang.instrument.Instrumentation;
import java.time.Duration;
import java.time.Instant;
public class HunkParserTest {
@@ -12,12 +20,20 @@ public class HunkParserTest {
public void testSimple() throws IOException {
// String input = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/revFiles/7f52f3_3845d29_drivers#pci#host#pcie-altera.c";
String root = "/Users/anilkoyuncu/projects/gumInputLinux/linux/";
String root = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/linux/";
// String filename = "8dd302_c4ef85_net#core#dev_ioctl.c";
String filename = "6a28fd_93ad867_drivers#tty#goldfish.c";
// String filename = "8dd302_c4ef85_net#core#dev_ioctl.c"; //ok
// String filename = "1e793f6_77f18a_drivers#scsi#megaraid#megaraid_sas_base.c"; //OK
// String filename = "6a28fd_93ad867_drivers#tty#goldfish.c"; //m,issing
// String filename = "b90f7c_ff51ff_kernel#sched#fair.c"; //wrong and wired
// String filename = "ed8f68_b1c8047_fs#ext3#dir.c";//ok
// String filename = "bc3d12_9a26653_drivers#scsi#libfc#fc_disc.c"; //okish
String filename = "118154_0c5f81_arch#x86#kvm#svm.c";
// String filename = "bcbd94f_43e43c9_drivers#md#dm-crypt.c"; //emin degilim
// String filename = "f1727b4_6c1e7e_arch#x86#kvm#vmx#nested.c"; //komplex not sure
// String filename = "5924f17_5925a05_net#ipv4#tcp.c";
// String filename = "bd0b9ac_b237721_drivers#irqchip#irq-dw-apb-ictl.c";
// String filename = "bd0b9ac_b237721_drivers#irqchip#irq-dw-apb-ictl.c"; //missing
// String filename = "052831_3985e8_include#net#ip_tunnels.h";
// String filename = "e76019_28647b_drivers#gpu#drm#i915#i915_drv.h";
// String filename = "4cbe4d_b124f4_include#linux#mlx4#device.h"; //enum case stops at block
@@ -26,10 +42,11 @@ public class HunkParserTest {
File prevFile =new File(root + "prevFiles/prev_"+filename);
EDiffHunkParser parser = new EDiffHunkParser();
String srcMLPath = "/Users/anilkoyuncu/Downloads/srcML2/src2srcml";
String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML/src2srcml";
parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath);
// ITree t = new SrcmlCppTreeGenerator().generateFromFile(input).getRoot();
// Assert.assertEquals(148, t.getSize());
}
@Test
@@ -42,7 +59,7 @@ public class HunkParserTest {
File prevFile =new File(root + "prevFiles/prev_"+filename);
File diffFile = new File(root + "DiffEntries/"+filename+".txt");
EDiffHunkParser parser = new EDiffHunkParser();
parser.parseFixPatterns(prevFile,revFile,diffFile,"gumInputLinux",null,srcMLPath);
parser.parseFixPatterns(prevFile,revFile,diffFile,"gumInputLinux",null,srcMLPath,null);
}
@@ -64,4 +81,46 @@ public class HunkParserTest {
// Assert.assertEquals(148, t.getSize());
}
@Test
public void rich() throws IOException {
Instant start = Instant.now();
final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "localhost",Integer.valueOf("6399"),20000000);
EDiffHunkParser parser = new EDiffHunkParser();
String root = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/linux/";
String filename = "bb67dd_0922c7_sound#soc#sof#intel#hda.c";
File revFile = new File(root + "revFiles/"+ filename);
File prevFile =new File(root + "prevFiles/prev_"+filename);
File diffFile = new File(root + "DiffEntries/"+filename+".txt");
String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML/src2srcml";
parser.parseFixPatterns(prevFile,revFile,diffFile,"gumInputLinux",outerPool,srcMLPath,"if");
String key = "if/3/linux_bb67dd_0922c7_sound#soc#sof#intel#hda.c.txt_0";
File file2load = new File("/Users/anil.koyuncu/projects/test/fixminer-core/python/data/dumps/"+ key);
byte[] dump = FileUtils.readFileToByteArray(file2load);
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(dump);
actionSet.toString();
// CODE HERE
Instant finish = Instant.now();
long timeElapsed = Duration.between(start, finish).toMillis();
System.out.println(timeElapsed);
// try (Jedis outer = outerPool.getResource()) {
// try {
//
//
//
// byte[] s = outer.hget("dump".getBytes(), key.getBytes());
// HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(s);
// actionSet.getActionSize();
// } catch (Exception e) {
// e.printStackTrace();
// }
// }
}
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java
+1 -1
View File
@@ -20,7 +20,7 @@ public abstract class Parser implements ParserInterface {
protected String originalTree = ""; // Guide of generating patches.
protected String actionSets = ""; // Guide of generating patches.
public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool, String srcMLPath);
public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, JedisPool innerPool, String srcMLPath,String rootType);
@Override
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java
+5 -3
View File
@@ -13,7 +13,8 @@ public class RunnableParser implements Runnable {
private String project;
private JedisPool pool;
private String srcMLPath;
private String rootType;
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) {
this.prevFile = prevFile;
this.revFile = revFile;
@@ -30,7 +31,7 @@ public class RunnableParser implements Runnable {
this.pool = innerPool;
}
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool,String srcMLPath) {
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool,String srcMLPath,String rootType) {
this.prevFile = prevFile;
this.revFile = revFile;
this.diffentryFile = diffentryFile;
@@ -38,10 +39,11 @@ public class RunnableParser implements Runnable {
this.project = project;
this.pool = innerPool;
this.srcMLPath = srcMLPath;
this.rootType = rootType;
}
@Override
public void run() {
parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,pool,srcMLPath);
parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,pool,srcMLPath,rootType);
}
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
+79 -9
View File
@@ -2,18 +2,20 @@ package edu.lu.uni.serval.fixminer.jobs;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage;
import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile;
import edu.lu.uni.serval.fixminer.akka.ediff.*;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.io.File;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -24,7 +26,7 @@ public class EnhancedASTDiff {
private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class);
public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath) throws Exception {
public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType) throws Exception {
String parameters = String.format("\nInput path %s",inputPath);
@@ -32,7 +34,11 @@ public class EnhancedASTDiff {
CallShell cs = new CallShell();
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
if (rootType == null){
cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
}else{
cmd = String.format(cmd, dbDir,rootType+chunkName,Integer.valueOf(portInner));
}
cs.runShell(cmd, portInner);
@@ -64,7 +70,7 @@ public class EnhancedASTDiff {
ActorSystem system = null;
ActorRef parsingActor = null;
final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath);
final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath,rootType);
try {
log.info("Akka begins...");
log.info("{} files to process ...", allMessageFiles.size());
@@ -88,7 +94,7 @@ public class EnhancedASTDiff {
forEach(m ->
{
EDiffHunkParser parser = new EDiffHunkParser();
parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool,"/Users/anilkoyuncu/Downloads/srcML2/src2srcml");
parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool,"/Users/anilkoyuncu/Downloads/srcML2/src2srcml",null);
if (counter % 10 == 0) {
log.info("Finalized parsing " + counter + " files... remaining " + (allMessageFiles.size() - counter));
}
@@ -141,5 +147,69 @@ public class EnhancedASTDiff {
}
}
public static void load(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType) throws Exception {
String parameters = String.format("\nInput path %s",inputPath);
log.info(parameters);
CallShell cs = new CallShell();
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
// if (rootType == null){
// cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
// }else{
// cmd = String.format(cmd, dbDir,rootType+chunkName,Integer.valueOf(portInner));
// }
cs.runShell(cmd, portInner);
JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000);
try (Jedis inner = innerPool.getResource()) {
inner.flushAll();
}
File folder = new File(new File(inputPath).getParent() + "/dumps/" + rootType);
File[] listOfFiles = folder.listFiles();
Stream<File> stream = Arrays.stream(listOfFiles);
List<File> folders = stream
.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
List<File> allMessageFiles = new ArrayList<>();
for (File target : folders) {
if(target.getName().startsWith("."))
continue;
List<File> files = Arrays.asList(target.listFiles());
if (files.size() > 1){
allMessageFiles.addAll(files);
}
}
log.info("Message size: "+allMessageFiles.size());
allMessageFiles.stream().
parallel().
forEach(x-> loadCore(x,innerPool));
}
public static void loadCore(File file2load, JedisPool innerPool){
try (Jedis inner = innerPool.getResource()) {
// byte[] dump = Files.readAllBytes(Paths.get(file2load.getPath()));
byte[] dump = FileUtils.readFileToByteArray(file2load);
// HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(dump);
String key = file2load.getPath().split("/dumps/")[1];
inner.hset("dump".getBytes(),key.getBytes(),dump);
// actionSet.toString();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
}
src/main/resource/app.properties
+4 -4
View File
@@ -3,11 +3,11 @@ pjName = gumInput
portInner = 6380
portDumps = 6399
parallelism = AKKA
numOfWorkers = 1
numOfWorkers = 14
inputPath = /Users/anilkoyuncu/projects/gumInputLinux/
redisPath = /Users/anilkoyuncu/projects/fixminer/fixminer-core/python/data/redis
srcMLPath= /Users/anilkoyuncu/Downloads/srcML2/src2srcml
inputPath = /Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux
redisPath = /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis
srcMLPath= /Users/anil.koyuncu/Downloads/srcML/src2srcml
actionType =ALL
eDiffTimeout = 900