azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #9 from SerVal-DTF/origin/dockerC

Browse Source 
Origin/docker c
This commit is contained in:
Anil Koyuncu
2020-03-05 15:03:37 +01:00
committed by GitHub
parent 475f04e4f4 6bdbdac4db
commit 596349e0a8
29 changed files with 410 additions and 786 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pom.xml
+8 -2
View File
@@ -30,6 +30,12 @@
<version>1.2</version>
</dependency>
<dependency>
<groupId>me.tongfei</groupId>
<artifactId>progressbar</artifactId>
<version>0.8.1</version>
</dependency>
<!-- akka -->
<dependency>
<groupId>com.typesafe.akka</groupId>
@@ -135,7 +141,7 @@
<archive>
<manifest>
<mainClass>
<!--edu.lu.uni.serval.fixminer.akka.compare.CompareTrees-->
<!--edu.lu.uni.serval.fixminer.jobs.CompareTrees-->
edu.lu.uni.serval.fixminer.Launcher
</mainClass>
</manifest>
@@ -152,6 +158,6 @@
</project>
<!--edu.lu.uni.serval.fixminer.Launcher
edu.lu.uni.serval.fixminer.akka.compare.CompareTrees
edu.lu.uni.serval.fixminer.jobs.CompareTrees
edu.lu.uni.serval.utils.ClusterToPattern
-->
src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+8 -12
View File
@@ -1,6 +1,6 @@
package edu.lu.uni.serval.fixminer;
import edu.lu.uni.serval.fixminer.akka.compare.CompareTrees;
import edu.lu.uni.serval.fixminer.jobs.CompareTrees;
import edu.lu.uni.serval.fixminer.jobs.EnhancedASTDiff;
import edu.lu.uni.serval.utils.ClusterToPattern;
import org.slf4j.Logger;
@@ -26,16 +26,16 @@ public class Launcher {
// String appConfigPath = args[0];
appProps.load(new FileInputStream(appConfigPath));
// String portInner = appProps.getProperty("portInner","6380");
String numOfWorkers = appProps.getProperty("numOfWorkers", "10");
String portDumps = appProps.getProperty("portDumps","6399");
String pjName = appProps.getProperty("pjName","allDataset");
String actionType = appProps.getProperty("actionType","ALL");
String eDiffTimeout = appProps.getProperty("eDiffTimeout","900");
String parallelism = appProps.getProperty("parallelism","FORKJOIN");
String hostname = appProps.getProperty("hostname","localhost");
String hunkLimit = appProps.getProperty("hunkLimit","10");
String patchSize = appProps.getProperty("patchSize","50");
String projectL = appProps.getProperty("projectList","");
String[] projectList = projectL.split(",");
String input = appProps.getProperty("inputPath","FORKJOIN");
String redisPath = appProps.getProperty("redisPath","FORKJOIN");
String srcMLPath = appProps.getProperty("srcMLPath","FORKJOIN");
@@ -58,12 +58,12 @@ public class Launcher {
//
// log.info(parameters);
mainLaunch( numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter, srcMLPath,hostname,hunkLimit);
mainLaunch( numOfWorkers, jobType, portDumps, pjName,actionType,input,redisPath,parameter, srcMLPath,hostname,hunkLimit,projectList,patchSize);
}
public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter,String srcMLPath,String hostname,String hunkLimit){
public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String input, String redisPath,String parameter,String srcMLPath,String hostname,String hunkLimit,String[] projectList,String patchSize){
String dbDir;
@@ -79,11 +79,7 @@ public class Launcher {
try {
switch (jobType) {
case "RICHEDITSCRIPT":
EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter,hunkLimit);
break;
case "LOAD":
EnhancedASTDiff.load(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter);
EnhancedASTDiff.main(gumInput, pjName, portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter,hunkLimit,projectList,patchSize);
break;
case "COMPARE":
src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java
-107
View File
@@ -1,107 +0,0 @@
package edu.lu.uni.serval.fixminer.akka.compare;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
import java.util.*;
import java.util.stream.Collectors;
/**
* Created by anilkoyuncu on 12/09/2018.
*/
public class AkkaTreeParser {
private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class);
public static List<String> getMessages(JedisPool innerPool, int cursor){
ScanResult<String> scan;
try (Jedis inner = innerPool.getResource()) {
while (!inner.ping().equals("PONG")){
log.info("wait");
}
ScanParams sc = new ScanParams();
//150000000
log.info("Scanning ");
sc.count(cursor);
sc.match("*");
// sc.match("pair_[0-9]*");
scan = inner.scan("0", sc);
int size = scan.getResult().size();
log.info("Scanned " + String.valueOf(size));
}
List<String> result = scan.getResult();
log.info("Getting results");
return result;
}
public static HashMap<String, String> filenames(JedisPool innerPool){
HashMap<String, String> fileMap =new HashMap<String, String>();
try (Jedis inner = innerPool.getResource()) {
while (!inner.ping().equals("PONG")){
log.info("wait");
}
inner.select(1);
Map<String, String> filenames = inner.hgetAll("filenames");
for (Map.Entry<String, String> stringStringEntry : filenames.entrySet().stream().collect(Collectors.toList())) {
fileMap.put(stringStringEntry.getKey(),stringStringEntry.getValue());
}
}
// log.info("Getting results %d",fileMap.s);
return fileMap;
}
public static List<String> files2compare(JedisPool innerPool){
// HashMap<String, String> fileMap =new HashMap<String, String>();
List<String> result = new ArrayList<String>();
try (Jedis inner = innerPool.getResource()) {
while (!inner.ping().equals("PONG")){
log.info("wait");
}
// inner.select(1);
Set<String> compare = inner.hkeys("compare");
// compare.size();
result= new ArrayList<String>(compare);
// for (Map.Entry<String, String> stringStringEntry : filenames.entrySet().stream().collect(Collectors.toList())) {
//// fileMap.put(stringStringEntry.getKey(),stringStringEntry.getValue());
// result.add(stringStringEntry.getKey());
// }
}
log.info("Getting results :" + result.size());
return result;
}
}
src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java
-421
View File
@@ -1,421 +0,0 @@
package edu.lu.uni.serval.fixminer.akka.compare;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.text.similarity.JaroWinklerDistance;
import org.javatuples.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
* Created by anilkoyuncu on 03/04/2018.
*/
public class CompareTrees {
private static Logger log = LoggerFactory.getLogger(CompareTrees.class);
public static void main(String redisPath, String portDumps, String dumpsName, String job,String numOfWorkers,String host) throws Exception {
// shape /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis ALLdumps-gumInput.rdb clusterl0-gumInputALL.rdb /Users/anil.koyuncu/projects/test/fixminer-core/python/data/richEditScript
// String portInner = "6380";
String port = portDumps; //"6399";
CallShell cs = new CallShell();
String cmd = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s";
cmd = String.format(cmd, redisPath,dumpsName,Integer.valueOf(port));
log.info(cmd);
cs.runShell(cmd, port);
// String cmdInner = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s";
// cmdInner = String.format(cmdInner, redisPath,compareDBName,Integer.valueOf(portInner));
// log.info(cmdInner);
// cs.runShell(cmdInner, portInner);
// String numOfWorkers = "100000000";//args[4];
// String host = "localhost";//args[5];
// -Djava.util.concurrent.ForkJoinPool.common.parallelism=256
// final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000);
final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(port),20000000);
// List<String> listOfPairs = AkkaTreeParser.getMessages(innerPool,Integer.valueOf(numOfWorkers));
HashMap<String, String> filenames = AkkaTreeParser.filenames(outerPool);
// List<String> listOfPairs = AkkaTreeParser.files2compare(outerPool);
ArrayList<String> samePairs = new ArrayList<>();
ArrayList<String> errorPairs = new ArrayList<>();
Integer numberOfWorkers = Integer.valueOf(numOfWorkers);
final ExecutorService executor = Executors.newWorkStealingPool(numberOfWorkers);
ArrayList<Future<?>> results = new ArrayList<Future<?>>();
for (int i = 1; i <numberOfWorkers ; i++) {
// schedule the work
log.info("Starting job {}",i);
final Future<?> future = executor.submit(new RunnableCompare(job, errorPairs, filenames, outerPool, i));
results.add(future);
}
for (Future<?> future:results){
try {
// wait for task to complete
future.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
} finally {
executor.shutdownNow();
}
}
log.info("End process");
}
public static class RunnableCompare implements Runnable {
String job;
ArrayList<String> errorPairs;
HashMap<String, String> filenames;
JedisPool outerPool;
Integer threadID;
public RunnableCompare(String treeType,ArrayList<String> errorPairs, HashMap<String, String> filenames,JedisPool outerPool,Integer threadID) {
this.job = treeType;
this.errorPairs = errorPairs;
this.filenames = filenames;
this.outerPool = outerPool;
this.threadID = threadID;
}
@Override
public void run() {
// int dbsize = 1;
boolean stop = true;
while(stop) {
// try (Jedis outer = outerPool.getResource()) {
// dbsize = Math.toIntExact(outer.scard("compare"));
// }
// if (dbsize != 0){
stop = newCoreCompare(job, errorPairs, filenames, outerPool);
// }
}
log.info("Completed worker {}",threadID);
}
}
public static boolean newCoreCompare( String treeType,ArrayList<String> errorPairs, HashMap<String, String> filenames,JedisPool outerPool ) {
String pairName;
try (Jedis outer = outerPool.getResource()) {
pairName = outer.spop("compare");
}
String matchKey = null;
try {
String[] split = pairName.split("/");
String i = split[1];
String j = split[2];
String keyName = split[0];
matchKey = keyName + "/" + (String.valueOf(i)) + "/" + String.valueOf(j);
switch (treeType) {
case "single":
if (matchKey == null){
return false;
}
Map<String, String> oldTreeString = EDiffHelper.getTreeString(keyName, i, outerPool, filenames);
Map<String, String> newTreeString = EDiffHelper.getTreeString(keyName, j, outerPool, filenames);
String oldShapeTree =oldTreeString.get("shapeTree");
String newShapeTree =newTreeString.get("shapeTree");
String oldActionTree = oldTreeString.get("actionTree");
String newActionTree = newTreeString.get("actionTree");
String oldTargetTree = oldTreeString.get("targetTree");
String newTargetTree = newTreeString.get("targetTree");
if (oldShapeTree.equals(newShapeTree)) {
if (oldActionTree.equals(newActionTree)) {
if (oldTargetTree.equals(newTargetTree)) {
// samePairs.add(matchKey);
try (Jedis jedis = outerPool.getResource()) {
//// jedis.del(matchKey);
jedis.select(2);
jedis.set(matchKey, "1");
}
}
}
}
return true;
default:
return true;
// break;
}
}catch (Exception e) {
errorPairs.add(matchKey);
if (pairName == null)
return false;
log.debug("{} not comparable", pairName);
}
return true;
}
// public static void coreCompare(String pairName, String treeType,JedisPool innerPool,ArrayList<String> samePairs,ArrayList<String> errorPairs, HashMap<String, String> filenames,JedisPool outerPool ) {
//
//// if (samePairs.size() % 1000 == 0) {
//// log.info("Same pairs size "+samePairs.size());
//// }
//
// ITree oldTree = null;
// ITree newTree = null;
// Pair<ITree, HierarchicalActionSet> oldPair = null;
// Pair<ITree, HierarchicalActionSet> newPair = null;
// String matchKey = null;
//
// innerPool = outerPool;
//
// try {
//
// String[] split = pairName.split("/");
//
//
// String i = split[1];
// String j = split[2];
// String keyName = split[0];
// matchKey = keyName + "/" + (String.valueOf(i)) + "/" + String.valueOf(j);
//// jedis.select(0);
//// Set<String> keys = jedis.keys(matchKey);
//// if (keys.size() > 0) {
//// jedis.del(matchKey);
//// } else {
//// return;
//// }
//// jedis.srem("pairs",matchKey);
//// JedisPool outerPool = null;
// switch (treeType) {
// case "single":
//
//// String oldShapeTree = EDiffHelper.getTreeString(keyName, i, outerPool, filenames,"shapeTree");
//// String newShapeTree = EDiffHelper.getTreeString(keyName, j, outerPool, filenames,"shapeTree");
////
//// String oldActionTree = EDiffHelper.getTreeString(keyName, i, outerPool, filenames,"actionTree");
//// String newActionTree = EDiffHelper.getTreeString(keyName, j, outerPool, filenames,"actionTree");
////
//// String oldTargetTree = EDiffHelper.getTreeString(keyName, i, outerPool, filenames,"targetTree");
//// String newTargetTree = EDiffHelper.getTreeString(keyName, j, outerPool, filenames,"targetTree");
////
////
//// if(oldShapeTree.equals(newShapeTree)){
//// if(oldActionTree.equals(newActionTree)){
//// if(oldTargetTree.equals(newTargetTree)){
//// samePairs.add(matchKey);
//// }
//// }
//// }
// return;
//// break;
//
//
// case "shape":
// oldTree = EDiffHelper.getShapes(keyName, i, outerPool,filenames);
// newTree = EDiffHelper.getShapes(keyName, j, outerPool,filenames);
// break;
// case "action":
//
// oldPair = EDiffHelper.getActions(keyName, i, outerPool, filenames);
// newPair = EDiffHelper.getActions(keyName, j, outerPool, filenames);
// oldTree = oldPair.getValue0();
// newTree = newPair.getValue0();
//
//
// break;
// case "token":
// oldTree = EDiffHelper.getTokens(keyName, i, outerPool, filenames);
// newTree = EDiffHelper.getTokens(keyName, j, outerPool, filenames);
//
// String oldTokens = EDiffHelper.getNames2(oldTree);
// String newTokens = EDiffHelper.getNames2(newTree);
//
// JaroWinklerDistance jwd = new JaroWinklerDistance();
//
//
// Double overallSimi = Double.valueOf(0);
//
// if (!(oldTokens.trim().isEmpty() || newTokens.trim().isEmpty())) {
// overallSimi = jwd.apply(oldTokens, newTokens);
//
// }
//
//
// int retval = Double.compare(overallSimi, Double.valueOf(1));
//
// if (retval >= 0) {
// String result = i + "," + j + "," + String.join(",", oldTokens);
//// jedis.select(2);
//// jedis.set(matchKey, result);
//// try (Jedis jedis = innerPool.getResource()) {
////// jedis.del(matchKey);
//// jedis.select(2);
//// jedis.set(matchKey, result);
//// }
// samePairs.add(matchKey);
//// try (Jedis jedis = innerPool.getResource()) {
////// jedis.del(matchKey);
//// jedis.select(2);
//// jedis.set(matchKey, result);
//// }
// }
//// jedis.select(0);
////// jedis.srem("pairs",matchKey);
//// jedis.del(matchKey);
//
// return;
// default:
// break;
// }
//
//
//
// if(oldTree.toStaticHashString().equals(newTree.toStaticHashString())){
// String editDistance = "0";
// String result = i + "," + j + "," + editDistance;
// if (editDistance.equals("0")) {
//
// if (treeType.equals("action")) {
// HierarchicalActionSet oldProject = oldPair.getValue1();
// HierarchicalActionSet newProject = newPair.getValue1();
//
// oldTree = EDiffHelper.getTargets(oldProject);
// newTree = EDiffHelper.getTargets(newProject);
// if (oldTree.toStaticHashString().equals(newTree.toStaticHashString())) {
// samePairs.add(matchKey);
//// try (Jedis jedis = innerPool.getResource()) {
////// jedis.del(matchKey);
//// jedis.select(2);
//// jedis.set(matchKey, result);
////
//// }
// }
// } else {
// samePairs.add(matchKey);
//// try (Jedis jedis = innerPool.getResource()) {
////// jedis.del(matchKey);
//// jedis.select(2);
//// jedis.set(matchKey, result);
////
//// }
// }
// }
//
//
// }
//// if(oldTree.toString().equals(newTree.toString())) {
//// Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
//// m.match();
////
////
//// ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings());
//// ag.generate();
//// List<Action> actions = ag.getActions();
////
////
//// String editDistance;
////
//// editDistance = String.valueOf(actions.size());
//// String result = i + "," + j + "," + editDistance;
////
////
//// if (editDistance.equals("0")) {
////
//// if (treeType.equals("action")) {
////
//// HierarchicalActionSet oldProject = oldPair.getValue1();
//// HierarchicalActionSet newProject = newPair.getValue1();
////
//// oldTree = EDiffHelper.getTargets(oldProject);
//// newTree = EDiffHelper.getTargets(newProject);
////
//// if(oldTree.toString().equals(newTree.toString())) {
//// m = Matchers.getInstance().getMatcher(oldTree, newTree);
//// m.match();
////
////
//// ag = new ActionGenerator(oldTree, newTree, m.getMappings());
//// ag.generate();
//// actions = ag.getActions();
////
//// editDistance = String.valueOf(actions.size());
////
//// if (editDistance.equals("0")) {
//// try (Jedis jedis = innerPool.getResource()) {
//// jedis.del(matchKey);
//// jedis.select(2);
//// jedis.set(matchKey, result);
//// }
//// }
//// }
////
////
//// } else {
//// try (Jedis jedis = innerPool.getResource()) {
//// jedis.del(matchKey);
//// jedis.select(2);
//// jedis.set(matchKey, result);
//// }
////// jedis.select(2);
////// jedis.set(matchKey, result);
////// samePairs.add(matchKey);
//// }
////
//// }
//// }
//
// } catch (Exception e) {
// errorPairs.add(matchKey);
//// jedis.select(0);
////// jedis.srem("pairs",matchKey);
////
//// jedis.hset(matchKey, "0", "1");
//
// log.debug("{} not comparable", pairName);
// }
// }
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/KryoContext.java
-10
View File
@@ -1,10 +0,0 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
public interface KryoContext {
byte[] serialze(Object obj);
byte[] serialze(Object obj, int bufferSize);
Object deserialze(Class clazz, byte[] serialized);
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffActor.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import akka.actor.ActorRef;
import akka.actor.Props;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffHunkParser.java
+24 -20
View File
@@ -1,17 +1,13 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.utils.EDiffHelper;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.ObjectOutputStream;
import java.util.List;
@@ -27,18 +23,27 @@ public class EDiffHunkParser extends EDiffParser {
private static Logger logger = LoggerFactory.getLogger(EDiffHunkParser.class);
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath,String hunkLimit) {
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile,srcMLPath);
String datasetName = project;
String[] split1 = diffentryFile.getParent().split(datasetName);
String root = split1[0];
String pj = split1[1].split("/")[1];
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath);
if (actionSets != null && actionSets.size() != 0) {
boolean processActionSet = true;
if (actionSets.size() > Integer.valueOf(hunkLimit)){
if (actionSets.size() > Integer.valueOf(hunkLimit)) {
processActionSet = false;
logger.debug("Skipping {} set size {}",diffentryFile.getName(),hunkLimit);
logger.debug("Skipping {} set size {}", diffentryFile.getName(), hunkLimit);
}
int hunkSet = 0;
if(processActionSet){
if (processActionSet) {
for (HierarchicalActionSet actionSet : actionSets) {
// FileOutputStream f = null;
@@ -51,23 +56,18 @@ public class EDiffHunkParser extends EDiffParser {
actionSet.toString();
int size = actionSet.getActionSize();
String datasetName = project;
String[] split1 = diffentryFile.getParent().split(datasetName);
String root = split1[0];
String pj = split1[1].split("/")[1];
String key = astNodeType+"/"+String.valueOf(size)+"/" + pj +"_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
String key = astNodeType + "/" + String.valueOf(size) + "/" + pj + "_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet);
ITree targetTree = EDiffHelper.getTargets(actionSet);
ITree actionTree = EDiffHelper.getActionTrees(actionSet);
ITree shapeTree = EDiffHelper.getShapeTree(actionSet);
try (Jedis inner = innerPool.getResource()) {
inner.hset("dump",key,actionSet.toString());
inner.hset(key,"actionTree",actionTree.toStaticHashString());
inner.hset(key,"targetTree",targetTree.toStaticHashString());
inner.hset(key,"shapeTree",shapeTree.toStaticHashString());
inner.hset("dump", key, actionSet.toString());
inner.hset(key, "actionTree", actionTree.toStaticHashString());
inner.hset(key, "targetTree", targetTree.toStaticHashString());
inner.hset(key, "shapeTree", shapeTree.toStaticHashString());
}
// File f = new File(root+"dumps/"+astNodeType+"/"+String.valueOf(size)+"/");
// f.mkdirs();
@@ -83,15 +83,19 @@ public class EDiffHunkParser extends EDiffParser {
// oos.close();
} catch (Exception e) {
logger.error("error",e);
logger.error("error", e);
// e.printStackTrace();
}
hunkSet++;
}
try (Jedis inner = innerPool.getResource()) {
inner.hset("diffEntry", pj + "_" + diffentryFile.getName(), "1");
}
}
}
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffMessage.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import edu.lu.uni.serval.fixminer.akka.BaseMessage;
import redis.clients.jedis.JedisPool;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffParser.java
+2 -1
View File
@@ -1,10 +1,11 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.gen.srcml.GumTreeCComparer;
import edu.lu.uni.serval.gumtree.GumTreeComparer;
import edu.lu.uni.serval.utils.ListSorter;
import redis.clients.jedis.JedisPool;
import java.io.File;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffWorker.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import akka.actor.Props;
import akka.actor.UntypedActor;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalActionSet.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.tree.ITree;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalRegrouper.java
+2 -1
View File
@@ -1,8 +1,9 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.actions.model.*;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.utils.ASTNodeMap;
import edu.lu.uni.serval.utils.ListSorter;
import java.util.ArrayList;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouperForC.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalRegrouperForC.java
+2 -4
View File
@@ -1,17 +1,15 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.actions.model.*;
import com.github.gumtreediff.gen.srcml.NodeMap_new;
import com.github.gumtreediff.io.CNodeMap;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.gumtree.GumTreeComparer;
import edu.lu.uni.serval.utils.ListSorter;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* Regroup GumTree results to a hierarchical construction.
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/HunkParserTest.java
+2 -13
View File
@@ -1,30 +1,19 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.gen.srcml.NodeMap_new;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser;
import edu.lu.uni.serval.utils.ClusterToPattern;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.io.FileUtils;
import org.javatuples.Pair;
import org.junit.Assert;
import org.junit.Test;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.io.File;
import java.io.IOException;
import java.lang.instrument.Instrumentation;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class HunkParserTest {
@@ -246,7 +235,7 @@ public class HunkParserTest {
String keyName = "if-3";
String i = "2";
String j = "21";
HashMap<String, String> filenames = AkkaTreeParser.filenames(outerPool);
// HashMap<String, String> filenames = AkkaTreeParser.filenames(outerPool);
// oldPair = EDiffHelper.getActions(keyName, i, outerPool, filenames);
// newPair = EDiffHelper.getActions(keyName, j, outerPool, filenames);
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/MessageFile.java
+10 -2
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import java.io.File;
@@ -9,13 +9,21 @@ public class MessageFile {
private File diffEntryFile;
private File positionFile;
public MessageFile(File revFile, File prevFile, File diffEntryFile) {
private String project;
public MessageFile(File revFile, File prevFile, File diffEntryFile,String project) {
super();
this.revFile = revFile;
this.prevFile = prevFile;
this.diffEntryFile = diffEntryFile;
this.project = project;
}
public String getProject() { return project;}
public void setProject(String project) { this.project = project;}
public File getRevFile() {
return revFile;
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/Parser.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import redis.clients.jedis.JedisPool;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/ParserInterface.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
public interface ParserInterface {
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/RunnableParser.java
+9 -9
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import redis.clients.jedis.JedisPool;
@@ -8,33 +8,33 @@ public class RunnableParser implements Runnable {
private File prevFile;
private File revFile;
private File diffentryFile;
private File diffEntryFile;
private Parser parser;
private String project;
private JedisPool pool;
private String srcMLPath;
private String rootType;
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) {
public RunnableParser(File prevFile, File revFile, File diffEntryFile, Parser parser) {
this.prevFile = prevFile;
this.revFile = revFile;
this.diffentryFile = diffentryFile;
this.diffEntryFile = diffEntryFile;
this.parser = parser;
}
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool) {
public RunnableParser(File prevFile, File revFile, File diffEntryFile, Parser parser, String project, JedisPool innerPool) {
this.prevFile = prevFile;
this.revFile = revFile;
this.diffentryFile = diffentryFile;
this.diffEntryFile = diffEntryFile;
this.parser = parser;
this.project = project;
this.pool = innerPool;
}
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool,String srcMLPath,String rootType) {
public RunnableParser(File prevFile, File revFile, File diffEntryFile, Parser parser, String project, JedisPool innerPool,String srcMLPath,String rootType) {
this.prevFile = prevFile;
this.revFile = revFile;
this.diffentryFile = diffentryFile;
this.diffEntryFile = diffEntryFile;
this.parser = parser;
this.project = project;
this.pool = innerPool;
@@ -44,6 +44,6 @@ public class RunnableParser implements Runnable {
@Override
public void run() {
parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,pool,srcMLPath,rootType);
parser.parseFixPatterns(prevFile, revFile, diffEntryFile,project,pool,srcMLPath,rootType);
}
}
src/main/java/edu/lu/uni/serval/fixminer/ediff/TestInputCases.java
+50
View File
@@ -0,0 +1,50 @@
package edu.lu.uni.serval.fixminer.ediff;
import org.junit.Assert;
import org.junit.Test;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
public class TestInputCases {
@Test
public void test_189_B_17295034_17295064() throws IOException {
//cpython_b3a601_63d5c1_Objects#unicodeobject.c
//linux_659d8c_fd2a50a_tools#perf#builtin-kmem.c
//openssl_6a14fe7_0ff368_crypto#LPdir_win.c
//linux_d1dc69_cc2115c_sound#soc#codecs#wm8776.c
//linux_ff244c6_a0917e0_drivers#net#tun.c .
List<HierarchicalActionSet> hierarchicalActionSets = getHierarchicalActionSets("linux_cf36a65_fa6ce9_drivers#video#console#vgacon.c");
Assert.assertEquals(hierarchicalActionSets.size(),1);
Assert.assertEquals(hierarchicalActionSets.get(0).toString(),"");
}
public List<HierarchicalActionSet> getHierarchicalActionSets(String s) throws IOException {
Properties appProps = new Properties();
appProps.load(new FileInputStream("src/main/resource/app.properties"));
String srcMLPath = appProps.getProperty("srcMLPath", "FORKJOIN");
String root = appProps.getProperty("inputPath");
String project = s.split("_")[0];
root = root + "/"+project+"/";
String filename = s.replace(project+"_","");
File revFile = new File(root + "revFiles/" + filename);
File prevFile = new File(root + "prevFiles/prev_" + filename);
EDiffHunkParser parser = new EDiffHunkParser();
List<HierarchicalActionSet> hierarchicalActionSets = parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath);
return hierarchicalActionSets;
}
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestIntroClass.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/TestIntroClass.java
+1 -4
View File
@@ -1,7 +1,5 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.utils.EDiffHelper;
import org.apache.commons.lang3.StringUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -13,7 +11,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Optional;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestPredefinedCases.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/TestPredefinedCases.java
+2 -13
View File
@@ -1,23 +1,12 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser;
import edu.lu.uni.serval.utils.ClusterToPattern;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.io.FileUtils;
import org.javatuples.Pair;
import org.junit.Assert;
import org.junit.Test;
import redis.clients.jedis.JedisPool;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestRealCases.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/TestRealCases.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.utils.EDiffHelper;
src/main/java/edu/lu/uni/serval/fixminer/jobs/CompareTrees.java
Executable
+230
View File
@@ -0,0 +1,230 @@
package edu.lu.uni.serval.fixminer.jobs;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import me.tongfei.progressbar.ProgressBar;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
/**
* Created by anilkoyuncu on 03/04/2018.
*/
public class CompareTrees {
private static Logger log = LoggerFactory.getLogger(CompareTrees.class);
public static void main(String redisPath, String portDumps, String dumpsName, String job,String numOfWorkers,String host) throws Exception {
// shape /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis ALLdumps-gumInput.rdb clusterl0-gumInputALL.rdb /Users/anil.koyuncu/projects/test/fixminer-core/python/data/richEditScript
// String portInner = "6380";
String port = portDumps; //"6399";
CallShell cs = new CallShell();
String cmd = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s";
cmd = String.format(cmd, redisPath,dumpsName,Integer.valueOf(port));
log.info(cmd);
cs.runShell(cmd, port);
// String cmdInner = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s";
// cmdInner = String.format(cmdInner, redisPath,compareDBName,Integer.valueOf(portInner));
// log.info(cmdInner);
// cs.runShell(cmdInner, portInner);
// String numOfWorkers = "100000000";//args[4];
// String host = "localhost";//args[5];
// -Djava.util.concurrent.ForkJoinPool.common.parallelism=256
// final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000);
final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(port),20000000);
// List<String> listOfPairs = AkkaTreeParser.getMessages(innerPool,Integer.valueOf(numOfWorkers));
HashMap<String, String> filenames = getFilenames(outerPool);
// List<String> listOfPairs = AkkaTreeParser.files2compare(outerPool);
ArrayList<String> samePairs = new ArrayList<>();
ArrayList<String> errorPairs = new ArrayList<>();
Integer numberOfWorkers = Integer.valueOf(numOfWorkers);
final ExecutorService executor = Executors.newWorkStealingPool(numberOfWorkers);
ArrayList<Future<?>> results = new ArrayList<Future<?>>();
for (int i = 1; i <numberOfWorkers ; i++) {
// schedule the work
log.info("Starting job {}",i);
final Future<?> future = executor.submit(new RunnableCompare(job, errorPairs, filenames, outerPool, i));
results.add(future);
}
for (Future<?> future : ProgressBar.wrap(results, "Comparing")){
// for (Future<?> future:results){
try {
// wait for task to complete
future.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
// finally {
// executor.shutdownNow();
// }
}
executor.shutdownNow();
log.info("End process");
}
public static class RunnableCompare implements Runnable {
String job;
ArrayList<String> errorPairs;
HashMap<String, String> filenames;
JedisPool outerPool;
Integer threadID;
public RunnableCompare(String treeType,ArrayList<String> errorPairs, HashMap<String, String> filenames,JedisPool outerPool,Integer threadID) {
this.job = treeType;
this.errorPairs = errorPairs;
this.filenames = filenames;
this.outerPool = outerPool;
this.threadID = threadID;
}
@Override
public void run() {
// int dbsize = 1;
boolean stop = true;
while(stop) {
// try (Jedis outer = outerPool.getResource()) {
// dbsize = Math.toIntExact(outer.scard("compare"));
// }
// if (dbsize != 0){
stop = newCoreCompare(job, errorPairs, filenames, outerPool);
// }
}
log.info("Completed worker {}",threadID);
}
}
public static boolean newCoreCompare( String treeType,ArrayList<String> errorPairs, HashMap<String, String> filenames,JedisPool outerPool ) {
String pairName;
try (Jedis outer = outerPool.getResource()) {
pairName = outer.spop("compare");
}
String matchKey = null;
try {
String[] split = pairName.split("/");
String i = split[1];
String j = split[2];
String keyName = split[0];
matchKey = keyName + "/" + (String.valueOf(i)) + "/" + String.valueOf(j);
switch (treeType) {
case "single":
if (matchKey == null){
return false;
}
Map<String, String> oldTreeString = EDiffHelper.getTreeString(keyName, i, outerPool, filenames);
Map<String, String> newTreeString = EDiffHelper.getTreeString(keyName, j, outerPool, filenames);
String oldShapeTree =oldTreeString.get("shapeTree");
String newShapeTree =newTreeString.get("shapeTree");
String oldActionTree = oldTreeString.get("actionTree");
String newActionTree = newTreeString.get("actionTree");
String oldTargetTree = oldTreeString.get("targetTree");
String newTargetTree = newTreeString.get("targetTree");
if (oldShapeTree.equals(newShapeTree)) {
if (oldActionTree.equals(newActionTree)) {
if (oldTargetTree.equals(newTargetTree)) {
// samePairs.add(matchKey);
try (Jedis jedis = outerPool.getResource()) {
//// jedis.del(matchKey);
jedis.select(2);
jedis.set(matchKey, "1");
}
}
}
}
return true;
default:
return true;
// break;
}
}catch (Exception e) {
errorPairs.add(matchKey);
if (pairName == null)
return false;
log.debug("{} not comparable", pairName);
}
return true;
}
public static HashMap<String, String> getFilenames(JedisPool innerPool){
HashMap<String, String> fileMap =new HashMap<String, String>();
try (Jedis inner = innerPool.getResource()) {
while (!inner.ping().equals("PONG")){
log.info("wait");
}
inner.select(1);
Map<String, String> filenames = inner.hgetAll("filenames");
for (Map.Entry<String, String> stringStringEntry : filenames.entrySet().stream().collect(Collectors.toList())) {
fileMap.put(stringStringEntry.getKey(),stringStringEntry.getValue());
}
}
// log.info("Getting results %d",fileMap.s);
return fileMap;
}
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
+33 -140
View File
@@ -1,22 +1,21 @@
package edu.lu.uni.serval.fixminer.jobs;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import edu.lu.uni.serval.fixminer.akka.ediff.*;
import edu.lu.uni.serval.fixminer.ediff.EDiffHunkParser;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.fixminer.ediff.MessageFile;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.io.FileUtils;
import me.tongfei.progressbar.ProgressBar;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -27,7 +26,7 @@ public class EnhancedASTDiff {
private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class);
public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType,String hunkLimit) throws Exception {
public static void main(String inputPath, String project, String portInner, String dbDir, String chunkName,String srcMLPath,String parameter,String hunkLimit,String[] projectList,String patchSize) throws Exception {
String parameters = String.format("\nInput path %s",inputPath);
@@ -35,34 +34,19 @@ public class EnhancedASTDiff {
CallShell cs = new CallShell();
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
// if (rootType == null){
cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
// }else{
// cmd = String.format(cmd, dbDir,rootType+chunkName,Integer.valueOf(portInner));
// }
cs.runShell(cmd, portInner);
JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000);
if (rootType == "add"){
try (Jedis inner = innerPool.getResource()) {
inner.select(2);
inner.flushDB();
inner.select(1);
inner.flushDB();
inner.select(0);
inner.del("compare");
List<Predicate<File>> allPredicates = new ArrayList<Predicate<File>>();
for (String s : projectList) {
Predicate<File> predicate = x->x.getName().endsWith(s);
allPredicates.add(predicate);
}
}
Predicate<File> predicate1 = x -> x.getName().endsWith("libtiff");
Predicate<File> predicate2 = x -> x.getName().endsWith("php-src");
Predicate<File> predicate3 = x -> x.getName().endsWith("cpython");
Predicate<File> predicate4 = x -> x.getName().endsWith("wireshark");
Predicate<File> predicate5 = x -> x.getName().endsWith("gzip");
File folder = new File(inputPath);
File[] listOfFiles = folder.listFiles();
Stream<File> stream = Arrays.stream(listOfFiles);
@@ -70,15 +54,14 @@ public class EnhancedASTDiff {
.filter(x -> !x.getName().startsWith("."))
.filter(x -> !x.getName().startsWith("cocci"))
.filter(x -> !x.getName().endsWith(".index"))
// .filter(x -> x.getName().endsWith("libtiff"))
.filter(predicate1.or(predicate2).or(predicate3).or(predicate4).or(predicate5))
.filter(allPredicates.stream().reduce(x->false, Predicate::or))
.collect(Collectors.toList());
List<MessageFile> allMessageFiles = new ArrayList<>();
for (File target : folders) {
List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/",project,patchSize); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
// msgFiles = msgFiles.subList(0,3000);
if (msgFiles == null)
@@ -88,64 +71,31 @@ public class EnhancedASTDiff {
}
switch (parallelism){
case "AKKA":
ActorSystem system = null;
ActorRef parsingActor = null;
Map<String, String> diffEntry;
try (Jedis inner = innerPool.getResource()) {
diffEntry = inner.hgetAll("diffEntry");
final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath,hunkLimit);
try {
log.info("Akka begins...");
log.info("{} files to process ...", allMessageFiles.size());
system = ActorSystem.create("Mining-FixPattern-System");
parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor");
parsingActor.tell(msg, ActorRef.noSender());
} catch (Exception e) {
system.shutdown();
e.printStackTrace();
}finally {
system.awaitTermination();
// system.shutdown();
}
break;
case "FORKJOIN":
int counter = new Object() {
int counter = 0;
{
allMessageFiles.stream().
parallel().
peek(x -> counter++).
log.info("{} files to process ...", allMessageFiles.size());
if (diffEntry != null) {
log.info("{} files already process ...", diffEntry.size());
allMessageFiles = allMessageFiles.stream().filter(f -> !diffEntry.containsKey(f.getProject() + "_" + f.getDiffEntryFile().getName())).collect(Collectors.toList());
log.info("{} files to process ...", allMessageFiles.size());
}
ProgressBar.wrap(allMessageFiles.stream().
parallel(),"Task").
forEach(m ->
{
EDiffHunkParser parser = new EDiffHunkParser();
parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool,srcMLPath,hunkLimit);
if (counter % 10 == 0) {
log.info("Finalized parsing " + counter + " files... remaining " + (allMessageFiles.size() - counter));
}
}
);
}
}.counter;
log.info("Finished parsing {} files",counter);
break;
default:
log.error("Unknown parallelism {}", parallelism);
break;
}
}
private static List<MessageFile> getMessageFiles(String gumTreeInput) {
private static List<MessageFile> getMessageFiles(String gumTreeInput,String datasetName,String patchSize) {
String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile
File revFilesPath = new File(inputPath + "revFiles/");
log.info(revFilesPath.getPath());
@@ -167,12 +117,18 @@ public class EnhancedASTDiff {
int count = 0;
while (matcher.find())
count++;
if(count>51)
if(count>= Integer.valueOf(patchSize))
// if(count>201)
continue;
// if(FileHelper.readFile(diffentryFile).split("@@\\s\\-\\d+,*\\d*\\s\\+\\d+,*\\d*\\s@@").length > 2)
// continue;
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
// String datasetName = project;
String[] split1 = diffentryFile.getParent().split(datasetName);
String root = split1[0];
String pj = split1[1].split("/")[1];
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile,pj);
msgFiles.add(msgFile);
@@ -184,69 +140,6 @@ public class EnhancedASTDiff {
}
}
public static void load(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType) throws Exception {
String parameters = String.format("\nInput path %s",inputPath);
log.info(parameters);
CallShell cs = new CallShell();
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
// if (rootType == null){
// cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
// }else{
// cmd = String.format(cmd, dbDir,rootType+chunkName,Integer.valueOf(portInner));
// }
cs.runShell(cmd, portInner);
JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000);
try (Jedis inner = innerPool.getResource()) {
inner.flushAll();
}
File folder = new File(new File(inputPath).getParent() + "/dumps/" + rootType);
File[] listOfFiles = folder.listFiles();
Stream<File> stream = Arrays.stream(listOfFiles);
List<File> folders = stream
.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
List<File> allMessageFiles = new ArrayList<>();
for (File target : folders) {
if(target.getName().startsWith("."))
continue;
List<File> files = Arrays.asList(target.listFiles());
if (files.size() > 1){
allMessageFiles.addAll(files);
}
}
log.info("Message size: "+allMessageFiles.size());
allMessageFiles.stream().
parallel().
forEach(x-> loadCore(x,innerPool));
}
public static void loadCore(File file2load, JedisPool innerPool){
try (Jedis inner = innerPool.getResource()) {
// byte[] dump = Files.readAllBytes(Paths.get(file2load.getPath()));
byte[] dump = FileUtils.readFileToByteArray(file2load);
// HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(dump);
String key = file2load.getPath().split("/dumps/")[1];
inner.hset("dump".getBytes(),key.getBytes(),dump);
// actionSet.toString();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
}
src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java
-1
View File
@@ -1,6 +1,5 @@
package edu.lu.uni.serval.utils;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java
+1 -3
View File
@@ -5,10 +5,8 @@ import com.github.gumtreediff.gen.srcml.NodeMap_new;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import com.github.gumtreediff.tree.TreeUtils;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.fixminer.akka.ediff.KryoContext;
import edu.lu.uni.serval.fixminer.ediff.HierarchicalActionSet;
import org.apache.commons.lang3.SerializationUtils;
import org.javatuples.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/FileHelper.java → src/main/java/edu/lu/uni/serval/utils/FileHelper.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.utils;
import java.io.BufferedInputStream;
import java.io.BufferedWriter;
import java.io.File;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ListSorter.java → src/main/java/edu/lu/uni/serval/utils/ListSorter.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
package edu.lu.uni.serval.utils;
import java.util.ArrayList;
import java.util.Collections;
src/main/resource/app.properties
+10 -7
View File
@@ -1,20 +1,23 @@
pjName = gumInput
portInner = 6380
pjName = patches
portDumps = 6399
parallelism = AKKA
numOfWorkers = 14
hostname = localhost
hunkLimit = 10
patchSize = 50
projectList = libtiff,php-src,cpython,wireshark,gzip,gmp,lighttpd1.4,lighttpd2
#inputPath = /Users/anilkoyuncu/projects/gumInputLinux
inputPath = /Users/anil.koyuncu/projects/fixminer/fixminer-data/gumInputLinux
#inputPath = /Users/anil.koyuncu/projects/fixminer/fixminer-data/gumInputLinux
inputPath = /Users/anil.koyuncu/projects/test/fixminer-data/patches
#redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
srcMLPath= /Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml
#redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
redisPath = /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis
#srcMLPath= /Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml
srcMLPath= /Users/anil.koyuncu/projects/test/srcML/bin/srcml
#srcMLPath= /usr/local/bin/srcml
actionType = ALL
eDiffTimeout = 900
#ENHANCEDASTDIFF,CACHE,LEVEL1,LEVEL2,LEVEL3