azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactors

Browse Source
This commit is contained in:
mimic
2020-03-05 15:02:11 +01:00
parent 131087a3f1
commit 6bdbdac4db
28 changed files with 318 additions and 610 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pom.xml
+2 -2
View File
@@ -141,7 +141,7 @@
<archive> <archive>
<manifest> <manifest>
<mainClass> <mainClass>
<!--edu.lu.uni.serval.fixminer.akka.compare.CompareTrees--> <!--edu.lu.uni.serval.fixminer.jobs.CompareTrees-->
edu.lu.uni.serval.fixminer.Launcher edu.lu.uni.serval.fixminer.Launcher
</mainClass> </mainClass>
</manifest> </manifest>
@@ -158,6 +158,6 @@
</project> </project>
<!--edu.lu.uni.serval.fixminer.Launcher <!--edu.lu.uni.serval.fixminer.Launcher
edu.lu.uni.serval.fixminer.akka.compare.CompareTrees edu.lu.uni.serval.fixminer.jobs.CompareTrees
edu.lu.uni.serval.utils.ClusterToPattern edu.lu.uni.serval.utils.ClusterToPattern
--> -->
src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+1 -1
View File
@@ -1,6 +1,6 @@
package edu.lu.uni.serval.fixminer; package edu.lu.uni.serval.fixminer;
import edu.lu.uni.serval.fixminer.akka.compare.CompareTrees; import edu.lu.uni.serval.fixminer.jobs.CompareTrees;
import edu.lu.uni.serval.fixminer.jobs.EnhancedASTDiff; import edu.lu.uni.serval.fixminer.jobs.EnhancedASTDiff;
import edu.lu.uni.serval.utils.ClusterToPattern; import edu.lu.uni.serval.utils.ClusterToPattern;
import org.slf4j.Logger; import org.slf4j.Logger;
src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java
-107
View File
@@ -1,107 +0,0 @@
package edu.lu.uni.serval.fixminer.akka.compare;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
import java.util.*;
import java.util.stream.Collectors;
/**
* Created by anilkoyuncu on 12/09/2018.
*/
public class AkkaTreeParser {
private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class);
public static List<String> getMessages(JedisPool innerPool, int cursor){
ScanResult<String> scan;
try (Jedis inner = innerPool.getResource()) {
while (!inner.ping().equals("PONG")){
log.info("wait");
}
ScanParams sc = new ScanParams();
//150000000
log.info("Scanning ");
sc.count(cursor);
sc.match("*");
// sc.match("pair_[0-9]*");
scan = inner.scan("0", sc);
int size = scan.getResult().size();
log.info("Scanned " + String.valueOf(size));
}
List<String> result = scan.getResult();
log.info("Getting results");
return result;
}
public static HashMap<String, String> filenames(JedisPool innerPool){
HashMap<String, String> fileMap =new HashMap<String, String>();
try (Jedis inner = innerPool.getResource()) {
while (!inner.ping().equals("PONG")){
log.info("wait");
}
inner.select(1);
Map<String, String> filenames = inner.hgetAll("filenames");
for (Map.Entry<String, String> stringStringEntry : filenames.entrySet().stream().collect(Collectors.toList())) {
fileMap.put(stringStringEntry.getKey(),stringStringEntry.getValue());
}
}
// log.info("Getting results %d",fileMap.s);
return fileMap;
}
public static List<String> files2compare(JedisPool innerPool){
// HashMap<String, String> fileMap =new HashMap<String, String>();
List<String> result = new ArrayList<String>();
try (Jedis inner = innerPool.getResource()) {
while (!inner.ping().equals("PONG")){
log.info("wait");
}
// inner.select(1);
Set<String> compare = inner.hkeys("compare");
// compare.size();
result= new ArrayList<String>(compare);
// for (Map.Entry<String, String> stringStringEntry : filenames.entrySet().stream().collect(Collectors.toList())) {
//// fileMap.put(stringStringEntry.getKey(),stringStringEntry.getValue());
// result.add(stringStringEntry.getKey());
// }
}
log.info("Getting results :" + result.size());
return result;
}
}
src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java
-423
View File
@@ -1,423 +0,0 @@
package edu.lu.uni.serval.fixminer.akka.compare;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import me.tongfei.progressbar.ProgressBar;
import org.apache.commons.text.similarity.JaroWinklerDistance;
import org.javatuples.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
* Created by anilkoyuncu on 03/04/2018.
*/
public class CompareTrees {
private static Logger log = LoggerFactory.getLogger(CompareTrees.class);
public static void main(String redisPath, String portDumps, String dumpsName, String job,String numOfWorkers,String host) throws Exception {
// shape /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis ALLdumps-gumInput.rdb clusterl0-gumInputALL.rdb /Users/anil.koyuncu/projects/test/fixminer-core/python/data/richEditScript
// String portInner = "6380";
String port = portDumps; //"6399";
CallShell cs = new CallShell();
String cmd = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s";
cmd = String.format(cmd, redisPath,dumpsName,Integer.valueOf(port));
log.info(cmd);
cs.runShell(cmd, port);
// String cmdInner = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s";
// cmdInner = String.format(cmdInner, redisPath,compareDBName,Integer.valueOf(portInner));
// log.info(cmdInner);
// cs.runShell(cmdInner, portInner);
// String numOfWorkers = "100000000";//args[4];
// String host = "localhost";//args[5];
// -Djava.util.concurrent.ForkJoinPool.common.parallelism=256
// final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000);
final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(port),20000000);
// List<String> listOfPairs = AkkaTreeParser.getMessages(innerPool,Integer.valueOf(numOfWorkers));
HashMap<String, String> filenames = AkkaTreeParser.filenames(outerPool);
// List<String> listOfPairs = AkkaTreeParser.files2compare(outerPool);
ArrayList<String> samePairs = new ArrayList<>();
ArrayList<String> errorPairs = new ArrayList<>();
Integer numberOfWorkers = Integer.valueOf(numOfWorkers);
final ExecutorService executor = Executors.newWorkStealingPool(numberOfWorkers);
ArrayList<Future<?>> results = new ArrayList<Future<?>>();
for (int i = 1; i <numberOfWorkers ; i++) {
// schedule the work
log.info("Starting job {}",i);
final Future<?> future = executor.submit(new RunnableCompare(job, errorPairs, filenames, outerPool, i));
results.add(future);
}
for (Future<?> future : ProgressBar.wrap(results, "Comparing")){
// for (Future<?> future:results){
try {
// wait for task to complete
future.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
// finally {
// executor.shutdownNow();
// }
}
executor.shutdownNow();
log.info("End process");
}
public static class RunnableCompare implements Runnable {
String job;
ArrayList<String> errorPairs;
HashMap<String, String> filenames;
JedisPool outerPool;
Integer threadID;
public RunnableCompare(String treeType,ArrayList<String> errorPairs, HashMap<String, String> filenames,JedisPool outerPool,Integer threadID) {
this.job = treeType;
this.errorPairs = errorPairs;
this.filenames = filenames;
this.outerPool = outerPool;
this.threadID = threadID;
}
@Override
public void run() {
// int dbsize = 1;
boolean stop = true;
while(stop) {
// try (Jedis outer = outerPool.getResource()) {
// dbsize = Math.toIntExact(outer.scard("compare"));
// }
// if (dbsize != 0){
stop = newCoreCompare(job, errorPairs, filenames, outerPool);
// }
}
log.info("Completed worker {}",threadID);
}
}
public static boolean newCoreCompare( String treeType,ArrayList<String> errorPairs, HashMap<String, String> filenames,JedisPool outerPool ) {
String pairName;
try (Jedis outer = outerPool.getResource()) {
pairName = outer.spop("compare");
}
String matchKey = null;
try {
String[] split = pairName.split("/");
String i = split[1];
String j = split[2];
String keyName = split[0];
matchKey = keyName + "/" + (String.valueOf(i)) + "/" + String.valueOf(j);
switch (treeType) {
case "single":
if (matchKey == null){
return false;
}
Map<String, String> oldTreeString = EDiffHelper.getTreeString(keyName, i, outerPool, filenames);
Map<String, String> newTreeString = EDiffHelper.getTreeString(keyName, j, outerPool, filenames);
String oldShapeTree =oldTreeString.get("shapeTree");
String newShapeTree =newTreeString.get("shapeTree");
String oldActionTree = oldTreeString.get("actionTree");
String newActionTree = newTreeString.get("actionTree");
String oldTargetTree = oldTreeString.get("targetTree");
String newTargetTree = newTreeString.get("targetTree");
if (oldShapeTree.equals(newShapeTree)) {
if (oldActionTree.equals(newActionTree)) {
if (oldTargetTree.equals(newTargetTree)) {
// samePairs.add(matchKey);
try (Jedis jedis = outerPool.getResource()) {
//// jedis.del(matchKey);
jedis.select(2);
jedis.set(matchKey, "1");
}
}
}
}
return true;
default:
return true;
// break;
}
}catch (Exception e) {
errorPairs.add(matchKey);
if (pairName == null)
return false;
log.debug("{} not comparable", pairName);
}
return true;
}
// public static void coreCompare(String pairName, String treeType,JedisPool innerPool,ArrayList<String> samePairs,ArrayList<String> errorPairs, HashMap<String, String> filenames,JedisPool outerPool ) {
//
//// if (samePairs.size() % 1000 == 0) {
//// log.info("Same pairs size "+samePairs.size());
//// }
//
// ITree oldTree = null;
// ITree newTree = null;
// Pair<ITree, HierarchicalActionSet> oldPair = null;
// Pair<ITree, HierarchicalActionSet> newPair = null;
// String matchKey = null;
//
// innerPool = outerPool;
//
// try {
//
// String[] split = pairName.split("/");
//
//
// String i = split[1];
// String j = split[2];
// String keyName = split[0];
// matchKey = keyName + "/" + (String.valueOf(i)) + "/" + String.valueOf(j);
//// jedis.select(0);
//// Set<String> keys = jedis.keys(matchKey);
//// if (keys.size() > 0) {
//// jedis.del(matchKey);
//// } else {
//// return;
//// }
//// jedis.srem("pairs",matchKey);
//// JedisPool outerPool = null;
// switch (treeType) {
// case "single":
//
//// String oldShapeTree = EDiffHelper.getTreeString(keyName, i, outerPool, filenames,"shapeTree");
//// String newShapeTree = EDiffHelper.getTreeString(keyName, j, outerPool, filenames,"shapeTree");
////
//// String oldActionTree = EDiffHelper.getTreeString(keyName, i, outerPool, filenames,"actionTree");
//// String newActionTree = EDiffHelper.getTreeString(keyName, j, outerPool, filenames,"actionTree");
////
//// String oldTargetTree = EDiffHelper.getTreeString(keyName, i, outerPool, filenames,"targetTree");
//// String newTargetTree = EDiffHelper.getTreeString(keyName, j, outerPool, filenames,"targetTree");
////
////
//// if(oldShapeTree.equals(newShapeTree)){
//// if(oldActionTree.equals(newActionTree)){
//// if(oldTargetTree.equals(newTargetTree)){
//// samePairs.add(matchKey);
//// }
//// }
//// }
// return;
//// break;
//
//
// case "shape":
// oldTree = EDiffHelper.getShapes(keyName, i, outerPool,filenames);
// newTree = EDiffHelper.getShapes(keyName, j, outerPool,filenames);
// break;
// case "action":
//
// oldPair = EDiffHelper.getActions(keyName, i, outerPool, filenames);
// newPair = EDiffHelper.getActions(keyName, j, outerPool, filenames);
// oldTree = oldPair.getValue0();
// newTree = newPair.getValue0();
//
//
// break;
// case "token":
// oldTree = EDiffHelper.getTokens(keyName, i, outerPool, filenames);
// newTree = EDiffHelper.getTokens(keyName, j, outerPool, filenames);
//
// String oldTokens = EDiffHelper.getNames2(oldTree);
// String newTokens = EDiffHelper.getNames2(newTree);
//
// JaroWinklerDistance jwd = new JaroWinklerDistance();
//
//
// Double overallSimi = Double.valueOf(0);
//
// if (!(oldTokens.trim().isEmpty() || newTokens.trim().isEmpty())) {
// overallSimi = jwd.apply(oldTokens, newTokens);
//
// }
//
//
// int retval = Double.compare(overallSimi, Double.valueOf(1));
//
// if (retval >= 0) {
// String result = i + "," + j + "," + String.join(",", oldTokens);
//// jedis.select(2);
//// jedis.set(matchKey, result);
//// try (Jedis jedis = innerPool.getResource()) {
////// jedis.del(matchKey);
//// jedis.select(2);
//// jedis.set(matchKey, result);
//// }
// samePairs.add(matchKey);
//// try (Jedis jedis = innerPool.getResource()) {
////// jedis.del(matchKey);
//// jedis.select(2);
//// jedis.set(matchKey, result);
//// }
// }
//// jedis.select(0);
////// jedis.srem("pairs",matchKey);
//// jedis.del(matchKey);
//
// return;
// default:
// break;
// }
//
//
//
// if(oldTree.toStaticHashString().equals(newTree.toStaticHashString())){
// String editDistance = "0";
// String result = i + "," + j + "," + editDistance;
// if (editDistance.equals("0")) {
//
// if (treeType.equals("action")) {
// HierarchicalActionSet oldProject = oldPair.getValue1();
// HierarchicalActionSet newProject = newPair.getValue1();
//
// oldTree = EDiffHelper.getTargets(oldProject);
// newTree = EDiffHelper.getTargets(newProject);
// if (oldTree.toStaticHashString().equals(newTree.toStaticHashString())) {
// samePairs.add(matchKey);
//// try (Jedis jedis = innerPool.getResource()) {
////// jedis.del(matchKey);
//// jedis.select(2);
//// jedis.set(matchKey, result);
////
//// }
// }
// } else {
// samePairs.add(matchKey);
//// try (Jedis jedis = innerPool.getResource()) {
////// jedis.del(matchKey);
//// jedis.select(2);
//// jedis.set(matchKey, result);
////
//// }
// }
// }
//
//
// }
//// if(oldTree.toString().equals(newTree.toString())) {
//// Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
//// m.match();
////
////
//// ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings());
//// ag.generate();
//// List<Action> actions = ag.getActions();
////
////
//// String editDistance;
////
//// editDistance = String.valueOf(actions.size());
//// String result = i + "," + j + "," + editDistance;
////
////
//// if (editDistance.equals("0")) {
////
//// if (treeType.equals("action")) {
////
//// HierarchicalActionSet oldProject = oldPair.getValue1();
//// HierarchicalActionSet newProject = newPair.getValue1();
////
//// oldTree = EDiffHelper.getTargets(oldProject);
//// newTree = EDiffHelper.getTargets(newProject);
////
//// if(oldTree.toString().equals(newTree.toString())) {
//// m = Matchers.getInstance().getMatcher(oldTree, newTree);
//// m.match();
////
////
//// ag = new ActionGenerator(oldTree, newTree, m.getMappings());
//// ag.generate();
//// actions = ag.getActions();
////
//// editDistance = String.valueOf(actions.size());
////
//// if (editDistance.equals("0")) {
//// try (Jedis jedis = innerPool.getResource()) {
//// jedis.del(matchKey);
//// jedis.select(2);
//// jedis.set(matchKey, result);
//// }
//// }
//// }
////
////
//// } else {
//// try (Jedis jedis = innerPool.getResource()) {
//// jedis.del(matchKey);
//// jedis.select(2);
//// jedis.set(matchKey, result);
//// }
////// jedis.select(2);
////// jedis.set(matchKey, result);
////// samePairs.add(matchKey);
//// }
////
//// }
//// }
//
// } catch (Exception e) {
// errorPairs.add(matchKey);
//// jedis.select(0);
////// jedis.srem("pairs",matchKey);
////
//// jedis.hset(matchKey, "0", "1");
//
// log.debug("{} not comparable", pairName);
// }
// }
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/KryoContext.java
-10
View File
@@ -1,10 +0,0 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
public interface KryoContext {
byte[] serialze(Object obj);
byte[] serialze(Object obj, int bufferSize);
Object deserialze(Class clazz, byte[] serialized);
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffActor.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import akka.actor.ActorRef; import akka.actor.ActorRef;
import akka.actor.Props; import akka.actor.Props;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffHunkParser.java
+1 -5
View File
@@ -1,17 +1,13 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.utils.EDiffHelper; import edu.lu.uni.serval.utils.EDiffHelper;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
import java.io.File; import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.ObjectOutputStream;
import java.util.List; import java.util.List;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffMessage.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import edu.lu.uni.serval.fixminer.akka.BaseMessage; import edu.lu.uni.serval.fixminer.akka.BaseMessage;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffParser.java
+2 -1
View File
@@ -1,10 +1,11 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.gen.srcml.GumTreeCComparer; import com.github.gumtreediff.gen.srcml.GumTreeCComparer;
import edu.lu.uni.serval.gumtree.GumTreeComparer; import edu.lu.uni.serval.gumtree.GumTreeComparer;
import edu.lu.uni.serval.utils.ListSorter;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
import java.io.File; import java.io.File;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffWorker.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import akka.actor.Props; import akka.actor.Props;
import akka.actor.UntypedActor; import akka.actor.UntypedActor;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalActionSet.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.ITree;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalRegrouper.java
+2 -1
View File
@@ -1,8 +1,9 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.actions.model.*;
import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.utils.ASTNodeMap; import edu.lu.uni.serval.utils.ASTNodeMap;
import edu.lu.uni.serval.utils.ListSorter;
import java.util.ArrayList; import java.util.ArrayList;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouperForC.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalRegrouperForC.java
+2 -4
View File
@@ -1,17 +1,15 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.actions.model.*;
import com.github.gumtreediff.gen.srcml.NodeMap_new; import com.github.gumtreediff.gen.srcml.NodeMap_new;
import com.github.gumtreediff.io.CNodeMap;
import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.gumtree.GumTreeComparer; import edu.lu.uni.serval.utils.ListSorter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
/** /**
* Regroup GumTree results to a hierarchical construction. * Regroup GumTree results to a hierarchical construction.
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/HunkParserTest.java
+2 -13
View File
@@ -1,30 +1,19 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.gen.srcml.NodeMap_new;
import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser;
import edu.lu.uni.serval.utils.ClusterToPattern; import edu.lu.uni.serval.utils.ClusterToPattern;
import edu.lu.uni.serval.utils.EDiffHelper; import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder; import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.javatuples.Pair; import org.javatuples.Pair;
import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.lang.instrument.Instrumentation;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Duration; import java.time.Duration;
import java.time.Instant; import java.time.Instant;
import java.util.*; import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class HunkParserTest { public class HunkParserTest {
@@ -246,7 +235,7 @@ public class HunkParserTest {
String keyName = "if-3"; String keyName = "if-3";
String i = "2"; String i = "2";
String j = "21"; String j = "21";
HashMap<String, String> filenames = AkkaTreeParser.filenames(outerPool); // HashMap<String, String> filenames = AkkaTreeParser.filenames(outerPool);
// oldPair = EDiffHelper.getActions(keyName, i, outerPool, filenames); // oldPair = EDiffHelper.getActions(keyName, i, outerPool, filenames);
// newPair = EDiffHelper.getActions(keyName, j, outerPool, filenames); // newPair = EDiffHelper.getActions(keyName, j, outerPool, filenames);
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/MessageFile.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import java.io.File; import java.io.File;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/Parser.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/ParserInterface.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
public interface ParserInterface { public interface ParserInterface {
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/RunnableParser.java
+9 -9
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
@@ -8,33 +8,33 @@ public class RunnableParser implements Runnable {
private File prevFile; private File prevFile;
private File revFile; private File revFile;
private File diffentryFile; private File diffEntryFile;
private Parser parser; private Parser parser;
private String project; private String project;
private JedisPool pool; private JedisPool pool;
private String srcMLPath; private String srcMLPath;
private String rootType; private String rootType;
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) { public RunnableParser(File prevFile, File revFile, File diffEntryFile, Parser parser) {
this.prevFile = prevFile; this.prevFile = prevFile;
this.revFile = revFile; this.revFile = revFile;
this.diffentryFile = diffentryFile; this.diffEntryFile = diffEntryFile;
this.parser = parser; this.parser = parser;
} }
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool) { public RunnableParser(File prevFile, File revFile, File diffEntryFile, Parser parser, String project, JedisPool innerPool) {
this.prevFile = prevFile; this.prevFile = prevFile;
this.revFile = revFile; this.revFile = revFile;
this.diffentryFile = diffentryFile; this.diffEntryFile = diffEntryFile;
this.parser = parser; this.parser = parser;
this.project = project; this.project = project;
this.pool = innerPool; this.pool = innerPool;
} }
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool,String srcMLPath,String rootType) { public RunnableParser(File prevFile, File revFile, File diffEntryFile, Parser parser, String project, JedisPool innerPool,String srcMLPath,String rootType) {
this.prevFile = prevFile; this.prevFile = prevFile;
this.revFile = revFile; this.revFile = revFile;
this.diffentryFile = diffentryFile; this.diffEntryFile = diffEntryFile;
this.parser = parser; this.parser = parser;
this.project = project; this.project = project;
this.pool = innerPool; this.pool = innerPool;
@@ -44,6 +44,6 @@ public class RunnableParser implements Runnable {
@Override @Override
public void run() { public void run() {
parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,pool,srcMLPath,rootType); parser.parseFixPatterns(prevFile, revFile, diffEntryFile,project,pool,srcMLPath,rootType);
} }
} }
src/main/java/edu/lu/uni/serval/fixminer/ediff/TestInputCases.java
+50
View File
@@ -0,0 +1,50 @@
package edu.lu.uni.serval.fixminer.ediff;
import org.junit.Assert;
import org.junit.Test;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
public class TestInputCases {
@Test
public void test_189_B_17295034_17295064() throws IOException {
//cpython_b3a601_63d5c1_Objects#unicodeobject.c
//linux_659d8c_fd2a50a_tools#perf#builtin-kmem.c
//openssl_6a14fe7_0ff368_crypto#LPdir_win.c
//linux_d1dc69_cc2115c_sound#soc#codecs#wm8776.c
//linux_ff244c6_a0917e0_drivers#net#tun.c .
List<HierarchicalActionSet> hierarchicalActionSets = getHierarchicalActionSets("linux_cf36a65_fa6ce9_drivers#video#console#vgacon.c");
Assert.assertEquals(hierarchicalActionSets.size(),1);
Assert.assertEquals(hierarchicalActionSets.get(0).toString(),"");
}
public List<HierarchicalActionSet> getHierarchicalActionSets(String s) throws IOException {
Properties appProps = new Properties();
appProps.load(new FileInputStream("src/main/resource/app.properties"));
String srcMLPath = appProps.getProperty("srcMLPath", "FORKJOIN");
String root = appProps.getProperty("inputPath");
String project = s.split("_")[0];
root = root + "/"+project+"/";
String filename = s.replace(project+"_","");
File revFile = new File(root + "revFiles/" + filename);
File prevFile = new File(root + "prevFiles/prev_" + filename);
EDiffHunkParser parser = new EDiffHunkParser();
List<HierarchicalActionSet> hierarchicalActionSets = parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath);
return hierarchicalActionSets;
}
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestIntroClass.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/TestIntroClass.java
+1 -4
View File
@@ -1,7 +1,5 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.utils.EDiffHelper;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@@ -13,7 +11,6 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.List; import java.util.List;
import java.util.Optional;
import java.util.Properties; import java.util.Properties;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestPredefinedCases.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/TestPredefinedCases.java
+2 -13
View File
@@ -1,23 +1,12 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser;
import edu.lu.uni.serval.utils.ClusterToPattern;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.io.FileUtils;
import org.javatuples.Pair;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import redis.clients.jedis.JedisPool;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestRealCases.java → src/main/java/edu/lu/uni/serval/fixminer/ediff/TestRealCases.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.fixminer.ediff;
import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.utils.EDiffHelper; import edu.lu.uni.serval.utils.EDiffHelper;
src/main/java/edu/lu/uni/serval/fixminer/jobs/CompareTrees.java
Executable
+230
View File
@@ -0,0 +1,230 @@
package edu.lu.uni.serval.fixminer.jobs;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import me.tongfei.progressbar.ProgressBar;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
/**
* Created by anilkoyuncu on 03/04/2018.
*/
public class CompareTrees {
private static Logger log = LoggerFactory.getLogger(CompareTrees.class);
public static void main(String redisPath, String portDumps, String dumpsName, String job,String numOfWorkers,String host) throws Exception {
// shape /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis ALLdumps-gumInput.rdb clusterl0-gumInputALL.rdb /Users/anil.koyuncu/projects/test/fixminer-core/python/data/richEditScript
// String portInner = "6380";
String port = portDumps; //"6399";
CallShell cs = new CallShell();
String cmd = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s";
cmd = String.format(cmd, redisPath,dumpsName,Integer.valueOf(port));
log.info(cmd);
cs.runShell(cmd, port);
// String cmdInner = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s";
// cmdInner = String.format(cmdInner, redisPath,compareDBName,Integer.valueOf(portInner));
// log.info(cmdInner);
// cs.runShell(cmdInner, portInner);
// String numOfWorkers = "100000000";//args[4];
// String host = "localhost";//args[5];
// -Djava.util.concurrent.ForkJoinPool.common.parallelism=256
// final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000);
final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(port),20000000);
// List<String> listOfPairs = AkkaTreeParser.getMessages(innerPool,Integer.valueOf(numOfWorkers));
HashMap<String, String> filenames = getFilenames(outerPool);
// List<String> listOfPairs = AkkaTreeParser.files2compare(outerPool);
ArrayList<String> samePairs = new ArrayList<>();
ArrayList<String> errorPairs = new ArrayList<>();
Integer numberOfWorkers = Integer.valueOf(numOfWorkers);
final ExecutorService executor = Executors.newWorkStealingPool(numberOfWorkers);
ArrayList<Future<?>> results = new ArrayList<Future<?>>();
for (int i = 1; i <numberOfWorkers ; i++) {
// schedule the work
log.info("Starting job {}",i);
final Future<?> future = executor.submit(new RunnableCompare(job, errorPairs, filenames, outerPool, i));
results.add(future);
}
for (Future<?> future : ProgressBar.wrap(results, "Comparing")){
// for (Future<?> future:results){
try {
// wait for task to complete
future.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
// finally {
// executor.shutdownNow();
// }
}
executor.shutdownNow();
log.info("End process");
}
public static class RunnableCompare implements Runnable {
String job;
ArrayList<String> errorPairs;
HashMap<String, String> filenames;
JedisPool outerPool;
Integer threadID;
public RunnableCompare(String treeType,ArrayList<String> errorPairs, HashMap<String, String> filenames,JedisPool outerPool,Integer threadID) {
this.job = treeType;
this.errorPairs = errorPairs;
this.filenames = filenames;
this.outerPool = outerPool;
this.threadID = threadID;
}
@Override
public void run() {
// int dbsize = 1;
boolean stop = true;
while(stop) {
// try (Jedis outer = outerPool.getResource()) {
// dbsize = Math.toIntExact(outer.scard("compare"));
// }
// if (dbsize != 0){
stop = newCoreCompare(job, errorPairs, filenames, outerPool);
// }
}
log.info("Completed worker {}",threadID);
}
}
public static boolean newCoreCompare( String treeType,ArrayList<String> errorPairs, HashMap<String, String> filenames,JedisPool outerPool ) {
String pairName;
try (Jedis outer = outerPool.getResource()) {
pairName = outer.spop("compare");
}
String matchKey = null;
try {
String[] split = pairName.split("/");
String i = split[1];
String j = split[2];
String keyName = split[0];
matchKey = keyName + "/" + (String.valueOf(i)) + "/" + String.valueOf(j);
switch (treeType) {
case "single":
if (matchKey == null){
return false;
}
Map<String, String> oldTreeString = EDiffHelper.getTreeString(keyName, i, outerPool, filenames);
Map<String, String> newTreeString = EDiffHelper.getTreeString(keyName, j, outerPool, filenames);
String oldShapeTree =oldTreeString.get("shapeTree");
String newShapeTree =newTreeString.get("shapeTree");
String oldActionTree = oldTreeString.get("actionTree");
String newActionTree = newTreeString.get("actionTree");
String oldTargetTree = oldTreeString.get("targetTree");
String newTargetTree = newTreeString.get("targetTree");
if (oldShapeTree.equals(newShapeTree)) {
if (oldActionTree.equals(newActionTree)) {
if (oldTargetTree.equals(newTargetTree)) {
// samePairs.add(matchKey);
try (Jedis jedis = outerPool.getResource()) {
//// jedis.del(matchKey);
jedis.select(2);
jedis.set(matchKey, "1");
}
}
}
}
return true;
default:
return true;
// break;
}
}catch (Exception e) {
errorPairs.add(matchKey);
if (pairName == null)
return false;
log.debug("{} not comparable", pairName);
}
return true;
}
public static HashMap<String, String> getFilenames(JedisPool innerPool){
HashMap<String, String> fileMap =new HashMap<String, String>();
try (Jedis inner = innerPool.getResource()) {
while (!inner.ping().equals("PONG")){
log.info("wait");
}
inner.select(1);
Map<String, String> filenames = inner.hgetAll("filenames");
for (Map.Entry<String, String> stringStringEntry : filenames.entrySet().stream().collect(Collectors.toList())) {
fileMap.put(stringStringEntry.getKey(),stringStringEntry.getValue());
}
}
// log.info("Getting results %d",fileMap.s);
return fileMap;
}
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
+3 -3
View File
@@ -1,8 +1,8 @@
package edu.lu.uni.serval.fixminer.jobs; package edu.lu.uni.serval.fixminer.jobs;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser; import edu.lu.uni.serval.fixminer.ediff.EDiffHunkParser;
import edu.lu.uni.serval.fixminer.akka.ediff.FileHelper; import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile; import edu.lu.uni.serval.fixminer.ediff.MessageFile;
import edu.lu.uni.serval.utils.CallShell; import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.PoolBuilder; import edu.lu.uni.serval.utils.PoolBuilder;
import me.tongfei.progressbar.ProgressBar; import me.tongfei.progressbar.ProgressBar;
src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java
-1
View File
@@ -1,6 +1,5 @@
package edu.lu.uni.serval.utils; package edu.lu.uni.serval.utils;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;
src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java
+1 -3
View File
@@ -5,10 +5,8 @@ import com.github.gumtreediff.gen.srcml.NodeMap_new;
import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext; import com.github.gumtreediff.tree.TreeContext;
import com.github.gumtreediff.tree.TreeUtils; import com.github.gumtreediff.tree.TreeUtils;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import edu.lu.uni.serval.fixminer.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.fixminer.akka.ediff.KryoContext;
import org.apache.commons.lang3.SerializationUtils; import org.apache.commons.lang3.SerializationUtils;
import org.javatuples.Pair;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/FileHelper.java → src/main/java/edu/lu/uni/serval/utils/FileHelper.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.utils;
import java.io.BufferedInputStream; import java.io.BufferedInputStream;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.File; import java.io.File;
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ListSorter.java → src/main/java/edu/lu/uni/serval/utils/ListSorter.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.akka.ediff; package edu.lu.uni.serval.utils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;