diff --git a/pom.xml b/pom.xml index 30b2944..a893250 100755 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,12 @@ javatuples 1.2 + + + me.tongfei + progressbar + 0.8.1 + @@ -135,7 +141,7 @@ - + edu.lu.uni.serval.fixminer.Launcher @@ -152,6 +158,6 @@ diff --git a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java index 6f5d982..88d5c32 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java @@ -1,6 +1,6 @@ package edu.lu.uni.serval.fixminer; -import edu.lu.uni.serval.fixminer.akka.compare.CompareTrees; +import edu.lu.uni.serval.fixminer.jobs.CompareTrees; import edu.lu.uni.serval.fixminer.jobs.EnhancedASTDiff; import edu.lu.uni.serval.utils.ClusterToPattern; import org.slf4j.Logger; @@ -26,16 +26,16 @@ public class Launcher { // String appConfigPath = args[0]; appProps.load(new FileInputStream(appConfigPath)); -// String portInner = appProps.getProperty("portInner","6380"); String numOfWorkers = appProps.getProperty("numOfWorkers", "10"); String portDumps = appProps.getProperty("portDumps","6399"); String pjName = appProps.getProperty("pjName","allDataset"); String actionType = appProps.getProperty("actionType","ALL"); - String eDiffTimeout = appProps.getProperty("eDiffTimeout","900"); - String parallelism = appProps.getProperty("parallelism","FORKJOIN"); + String hostname = appProps.getProperty("hostname","localhost"); String hunkLimit = appProps.getProperty("hunkLimit","10"); - + String patchSize = appProps.getProperty("patchSize","50"); + String projectL = appProps.getProperty("projectList",""); + String[] projectList = projectL.split(","); String input = appProps.getProperty("inputPath","FORKJOIN"); String redisPath = appProps.getProperty("redisPath","FORKJOIN"); String srcMLPath = appProps.getProperty("srcMLPath","FORKJOIN"); @@ -58,12 +58,12 @@ public class Launcher { // // log.info(parameters); - mainLaunch( numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter, srcMLPath,hostname,hunkLimit); + mainLaunch( numOfWorkers, jobType, portDumps, pjName,actionType,input,redisPath,parameter, srcMLPath,hostname,hunkLimit,projectList,patchSize); } - public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter,String srcMLPath,String hostname,String hunkLimit){ + public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String input, String redisPath,String parameter,String srcMLPath,String hostname,String hunkLimit,String[] projectList,String patchSize){ String dbDir; @@ -79,11 +79,7 @@ public class Launcher { try { switch (jobType) { case "RICHEDITSCRIPT": - EnhancedASTDiff.main(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter,hunkLimit); - break; - - case "LOAD": - EnhancedASTDiff.load(gumInput, numOfWorkers, pjName, eDiffTimeout,parallelism,portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter); + EnhancedASTDiff.main(gumInput, pjName, portDumps, dbDir, actionType+dumpsName, srcMLPath,parameter,hunkLimit,projectList,patchSize); break; case "COMPARE": diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java deleted file mode 100755 index 8516f07..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java +++ /dev/null @@ -1,107 +0,0 @@ -package edu.lu.uni.serval.fixminer.akka.compare; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.ScanParams; -import redis.clients.jedis.ScanResult; - -import java.util.*; -import java.util.stream.Collectors; - -/** - * Created by anilkoyuncu on 12/09/2018. - */ -public class AkkaTreeParser { - - private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class); - - public static List getMessages(JedisPool innerPool, int cursor){ - - - ScanResult scan; - - try (Jedis inner = innerPool.getResource()) { - while (!inner.ping().equals("PONG")){ - log.info("wait"); - } - ScanParams sc = new ScanParams(); - //150000000 - log.info("Scanning "); - sc.count(cursor); - - sc.match("*"); -// sc.match("pair_[0-9]*"); - - scan = inner.scan("0", sc); - - int size = scan.getResult().size(); - log.info("Scanned " + String.valueOf(size)); - - } - List result = scan.getResult(); - log.info("Getting results"); - return result; - } - - public static HashMap filenames(JedisPool innerPool){ - - - HashMap fileMap =new HashMap(); - - try (Jedis inner = innerPool.getResource()) { - while (!inner.ping().equals("PONG")){ - log.info("wait"); - } - - inner.select(1); - Map filenames = inner.hgetAll("filenames"); - - - for (Map.Entry stringStringEntry : filenames.entrySet().stream().collect(Collectors.toList())) { - fileMap.put(stringStringEntry.getKey(),stringStringEntry.getValue()); - } - - - - - - } - -// log.info("Getting results %d",fileMap.s); - return fileMap; - } - - public static List files2compare(JedisPool innerPool){ - - -// HashMap fileMap =new HashMap(); - List result = new ArrayList(); - try (Jedis inner = innerPool.getResource()) { - while (!inner.ping().equals("PONG")){ - log.info("wait"); - } - -// inner.select(1); - Set compare = inner.hkeys("compare"); -// compare.size(); - result= new ArrayList(compare); - -// for (Map.Entry stringStringEntry : filenames.entrySet().stream().collect(Collectors.toList())) { -//// fileMap.put(stringStringEntry.getKey(),stringStringEntry.getValue()); -// result.add(stringStringEntry.getKey()); -// } - - - - - - } - - log.info("Getting results :" + result.size()); - return result; - } - - -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java deleted file mode 100755 index 796b487..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java +++ /dev/null @@ -1,421 +0,0 @@ -package edu.lu.uni.serval.fixminer.akka.compare; - -import com.github.gumtreediff.tree.ITree; -import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; -import edu.lu.uni.serval.utils.CallShell; -import edu.lu.uni.serval.utils.EDiffHelper; -import edu.lu.uni.serval.utils.PoolBuilder; -import org.apache.commons.text.similarity.JaroWinklerDistance; -import org.javatuples.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - - -/** - * Created by anilkoyuncu on 03/04/2018. - */ -public class CompareTrees { - - private static Logger log = LoggerFactory.getLogger(CompareTrees.class); - - - public static void main(String redisPath, String portDumps, String dumpsName, String job,String numOfWorkers,String host) throws Exception { - - // shape /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis ALLdumps-gumInput.rdb clusterl0-gumInputALL.rdb /Users/anil.koyuncu/projects/test/fixminer-core/python/data/richEditScript - -// String portInner = "6380"; - String port = portDumps; //"6399"; - CallShell cs = new CallShell(); - String cmd = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s"; - cmd = String.format(cmd, redisPath,dumpsName,Integer.valueOf(port)); - log.info(cmd); - cs.runShell(cmd, port); - -// String cmdInner = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s"; -// cmdInner = String.format(cmdInner, redisPath,compareDBName,Integer.valueOf(portInner)); -// log.info(cmdInner); -// cs.runShell(cmdInner, portInner); - -// String numOfWorkers = "100000000";//args[4]; -// String host = "localhost";//args[5]; -// -Djava.util.concurrent.ForkJoinPool.common.parallelism=256 - -// final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000); - - final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(port),20000000); - -// List listOfPairs = AkkaTreeParser.getMessages(innerPool,Integer.valueOf(numOfWorkers)); - HashMap filenames = AkkaTreeParser.filenames(outerPool); -// List listOfPairs = AkkaTreeParser.files2compare(outerPool); - - - ArrayList samePairs = new ArrayList<>(); - ArrayList errorPairs = new ArrayList<>(); - - Integer numberOfWorkers = Integer.valueOf(numOfWorkers); - final ExecutorService executor = Executors.newWorkStealingPool(numberOfWorkers); - ArrayList> results = new ArrayList>(); - for (int i = 1; i future = executor.submit(new RunnableCompare(job, errorPairs, filenames, outerPool, i)); - results.add(future); - } - - for (Future future:results){ - try { - // wait for task to complete - future.get(); - - } catch (InterruptedException e) { - - e.printStackTrace(); - } catch (ExecutionException e) { - - e.printStackTrace(); - } finally { - executor.shutdownNow(); - } - } - - - - - - log.info("End process"); - } - - - public static class RunnableCompare implements Runnable { - String job; - ArrayList errorPairs; - HashMap filenames; - JedisPool outerPool; - Integer threadID; - - public RunnableCompare(String treeType,ArrayList errorPairs, HashMap filenames,JedisPool outerPool,Integer threadID) { - this.job = treeType; - this.errorPairs = errorPairs; - this.filenames = filenames; - this.outerPool = outerPool; - this.threadID = threadID; - } - - @Override - public void run() { -// int dbsize = 1; - boolean stop = true; - while(stop) { -// try (Jedis outer = outerPool.getResource()) { -// dbsize = Math.toIntExact(outer.scard("compare")); -// } -// if (dbsize != 0){ - stop = newCoreCompare(job, errorPairs, filenames, outerPool); -// } - } - log.info("Completed worker {}",threadID); - } - } - - - public static boolean newCoreCompare( String treeType,ArrayList errorPairs, HashMap filenames,JedisPool outerPool ) { - - String pairName; - try (Jedis outer = outerPool.getResource()) { - pairName = outer.spop("compare"); - } - - String matchKey = null; - try { - - String[] split = pairName.split("/"); - - - String i = split[1]; - String j = split[2]; - String keyName = split[0]; - matchKey = keyName + "/" + (String.valueOf(i)) + "/" + String.valueOf(j); - - switch (treeType) { - case "single": - - if (matchKey == null){ - return false; - } - Map oldTreeString = EDiffHelper.getTreeString(keyName, i, outerPool, filenames); - Map newTreeString = EDiffHelper.getTreeString(keyName, j, outerPool, filenames); - - String oldShapeTree =oldTreeString.get("shapeTree"); - String newShapeTree =newTreeString.get("shapeTree"); - - String oldActionTree = oldTreeString.get("actionTree"); - String newActionTree = newTreeString.get("actionTree"); - - String oldTargetTree = oldTreeString.get("targetTree"); - String newTargetTree = newTreeString.get("targetTree"); - - - if (oldShapeTree.equals(newShapeTree)) { - if (oldActionTree.equals(newActionTree)) { - if (oldTargetTree.equals(newTargetTree)) { -// samePairs.add(matchKey); - try (Jedis jedis = outerPool.getResource()) { -//// jedis.del(matchKey); - jedis.select(2); - jedis.set(matchKey, "1"); - } - } - } - } - return true; - default: - return true; -// break; - } - }catch (Exception e) { - errorPairs.add(matchKey); - if (pairName == null) - return false; - log.debug("{} not comparable", pairName); - } - - - return true; - } - -// public static void coreCompare(String pairName, String treeType,JedisPool innerPool,ArrayList samePairs,ArrayList errorPairs, HashMap filenames,JedisPool outerPool ) { -// -//// if (samePairs.size() % 1000 == 0) { -//// log.info("Same pairs size "+samePairs.size()); -//// } -// -// ITree oldTree = null; -// ITree newTree = null; -// Pair oldPair = null; -// Pair newPair = null; -// String matchKey = null; -// -// innerPool = outerPool; -// -// try { -// -// String[] split = pairName.split("/"); -// -// -// String i = split[1]; -// String j = split[2]; -// String keyName = split[0]; -// matchKey = keyName + "/" + (String.valueOf(i)) + "/" + String.valueOf(j); -//// jedis.select(0); -//// Set keys = jedis.keys(matchKey); -//// if (keys.size() > 0) { -//// jedis.del(matchKey); -//// } else { -//// return; -//// } -//// jedis.srem("pairs",matchKey); -//// JedisPool outerPool = null; -// switch (treeType) { -// case "single": -// -//// String oldShapeTree = EDiffHelper.getTreeString(keyName, i, outerPool, filenames,"shapeTree"); -//// String newShapeTree = EDiffHelper.getTreeString(keyName, j, outerPool, filenames,"shapeTree"); -//// -//// String oldActionTree = EDiffHelper.getTreeString(keyName, i, outerPool, filenames,"actionTree"); -//// String newActionTree = EDiffHelper.getTreeString(keyName, j, outerPool, filenames,"actionTree"); -//// -//// String oldTargetTree = EDiffHelper.getTreeString(keyName, i, outerPool, filenames,"targetTree"); -//// String newTargetTree = EDiffHelper.getTreeString(keyName, j, outerPool, filenames,"targetTree"); -//// -//// -//// if(oldShapeTree.equals(newShapeTree)){ -//// if(oldActionTree.equals(newActionTree)){ -//// if(oldTargetTree.equals(newTargetTree)){ -//// samePairs.add(matchKey); -//// } -//// } -//// } -// return; -//// break; -// -// -// case "shape": -// oldTree = EDiffHelper.getShapes(keyName, i, outerPool,filenames); -// newTree = EDiffHelper.getShapes(keyName, j, outerPool,filenames); -// break; -// case "action": -// -// oldPair = EDiffHelper.getActions(keyName, i, outerPool, filenames); -// newPair = EDiffHelper.getActions(keyName, j, outerPool, filenames); -// oldTree = oldPair.getValue0(); -// newTree = newPair.getValue0(); -// -// -// break; -// case "token": -// oldTree = EDiffHelper.getTokens(keyName, i, outerPool, filenames); -// newTree = EDiffHelper.getTokens(keyName, j, outerPool, filenames); -// -// String oldTokens = EDiffHelper.getNames2(oldTree); -// String newTokens = EDiffHelper.getNames2(newTree); -// -// JaroWinklerDistance jwd = new JaroWinklerDistance(); -// -// -// Double overallSimi = Double.valueOf(0); -// -// if (!(oldTokens.trim().isEmpty() || newTokens.trim().isEmpty())) { -// overallSimi = jwd.apply(oldTokens, newTokens); -// -// } -// -// -// int retval = Double.compare(overallSimi, Double.valueOf(1)); -// -// if (retval >= 0) { -// String result = i + "," + j + "," + String.join(",", oldTokens); -//// jedis.select(2); -//// jedis.set(matchKey, result); -//// try (Jedis jedis = innerPool.getResource()) { -////// jedis.del(matchKey); -//// jedis.select(2); -//// jedis.set(matchKey, result); -//// } -// samePairs.add(matchKey); -//// try (Jedis jedis = innerPool.getResource()) { -////// jedis.del(matchKey); -//// jedis.select(2); -//// jedis.set(matchKey, result); -//// } -// } -//// jedis.select(0); -////// jedis.srem("pairs",matchKey); -//// jedis.del(matchKey); -// -// return; -// default: -// break; -// } -// -// -// -// if(oldTree.toStaticHashString().equals(newTree.toStaticHashString())){ -// String editDistance = "0"; -// String result = i + "," + j + "," + editDistance; -// if (editDistance.equals("0")) { -// -// if (treeType.equals("action")) { -// HierarchicalActionSet oldProject = oldPair.getValue1(); -// HierarchicalActionSet newProject = newPair.getValue1(); -// -// oldTree = EDiffHelper.getTargets(oldProject); -// newTree = EDiffHelper.getTargets(newProject); -// if (oldTree.toStaticHashString().equals(newTree.toStaticHashString())) { -// samePairs.add(matchKey); -//// try (Jedis jedis = innerPool.getResource()) { -////// jedis.del(matchKey); -//// jedis.select(2); -//// jedis.set(matchKey, result); -//// -//// } -// } -// } else { -// samePairs.add(matchKey); -//// try (Jedis jedis = innerPool.getResource()) { -////// jedis.del(matchKey); -//// jedis.select(2); -//// jedis.set(matchKey, result); -//// -//// } -// } -// } -// -// -// } -//// if(oldTree.toString().equals(newTree.toString())) { -//// Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); -//// m.match(); -//// -//// -//// ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); -//// ag.generate(); -//// List actions = ag.getActions(); -//// -//// -//// String editDistance; -//// -//// editDistance = String.valueOf(actions.size()); -//// String result = i + "," + j + "," + editDistance; -//// -//// -//// if (editDistance.equals("0")) { -//// -//// if (treeType.equals("action")) { -//// -//// HierarchicalActionSet oldProject = oldPair.getValue1(); -//// HierarchicalActionSet newProject = newPair.getValue1(); -//// -//// oldTree = EDiffHelper.getTargets(oldProject); -//// newTree = EDiffHelper.getTargets(newProject); -//// -//// if(oldTree.toString().equals(newTree.toString())) { -//// m = Matchers.getInstance().getMatcher(oldTree, newTree); -//// m.match(); -//// -//// -//// ag = new ActionGenerator(oldTree, newTree, m.getMappings()); -//// ag.generate(); -//// actions = ag.getActions(); -//// -//// editDistance = String.valueOf(actions.size()); -//// -//// if (editDistance.equals("0")) { -//// try (Jedis jedis = innerPool.getResource()) { -//// jedis.del(matchKey); -//// jedis.select(2); -//// jedis.set(matchKey, result); -//// } -//// } -//// } -//// -//// -//// } else { -//// try (Jedis jedis = innerPool.getResource()) { -//// jedis.del(matchKey); -//// jedis.select(2); -//// jedis.set(matchKey, result); -//// } -////// jedis.select(2); -////// jedis.set(matchKey, result); -////// samePairs.add(matchKey); -//// } -//// -//// } -//// } -// -// } catch (Exception e) { -// errorPairs.add(matchKey); -//// jedis.select(0); -////// jedis.srem("pairs",matchKey); -//// -//// jedis.hset(matchKey, "0", "1"); -// -// log.debug("{} not comparable", pairName); -// } -// } - - -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/KryoContext.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/KryoContext.java deleted file mode 100644 index d641ede..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/KryoContext.java +++ /dev/null @@ -1,10 +0,0 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; - -public interface KryoContext { - - byte[] serialze(Object obj); - - byte[] serialze(Object obj, int bufferSize); - - Object deserialze(Class clazz, byte[] serialized); -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffActor.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffActor.java index 425643a..342f058 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffActor.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; import akka.actor.ActorRef; import akka.actor.Props; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffHunkParser.java similarity index 68% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffHunkParser.java index bf77187..7c8f141 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffHunkParser.java @@ -1,24 +1,20 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; import com.github.gumtreediff.tree.ITree; import edu.lu.uni.serval.utils.EDiffHelper; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import java.io.File; -import java.io.FileOutputStream; -import java.io.FileWriter; -import java.io.ObjectOutputStream; import java.util.List; /** * Parse fix violations with GumTree in terms of multiple statements. - * + * * @author kui.liu * */ @@ -27,18 +23,27 @@ public class EDiffHunkParser extends EDiffParser { private static Logger logger = LoggerFactory.getLogger(EDiffHunkParser.class); @Override public void parseFixPatterns(File prevFile, File revFile, File diffentryFile, String project, JedisPool innerPool,String srcMLPath,String hunkLimit) { - List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile,srcMLPath); + + String datasetName = project; + String[] split1 = diffentryFile.getParent().split(datasetName); + String root = split1[0]; + String pj = split1[1].split("/")[1]; + + + List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath); + if (actionSets != null && actionSets.size() != 0) { boolean processActionSet = true; - if (actionSets.size() > Integer.valueOf(hunkLimit)){ + if (actionSets.size() > Integer.valueOf(hunkLimit)) { processActionSet = false; - logger.debug("Skipping {} set size {}",diffentryFile.getName(),hunkLimit); + logger.debug("Skipping {} set size {}", diffentryFile.getName(), hunkLimit); } int hunkSet = 0; - if(processActionSet){ + if (processActionSet) { + for (HierarchicalActionSet actionSet : actionSets) { // FileOutputStream f = null; @@ -51,23 +56,18 @@ public class EDiffHunkParser extends EDiffParser { actionSet.toString(); int size = actionSet.getActionSize(); - String datasetName = project; - String[] split1 = diffentryFile.getParent().split(datasetName); - String root = split1[0]; - String pj = split1[1].split("/")[1]; - - String key = astNodeType+"/"+String.valueOf(size)+"/" + pj +"_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet); + String key = astNodeType + "/" + String.valueOf(size) + "/" + pj + "_" + diffentryFile.getName() + "_" + String.valueOf(hunkSet); ITree targetTree = EDiffHelper.getTargets(actionSet); ITree actionTree = EDiffHelper.getActionTrees(actionSet); ITree shapeTree = EDiffHelper.getShapeTree(actionSet); try (Jedis inner = innerPool.getResource()) { - inner.hset("dump",key,actionSet.toString()); - inner.hset(key,"actionTree",actionTree.toStaticHashString()); - inner.hset(key,"targetTree",targetTree.toStaticHashString()); - inner.hset(key,"shapeTree",shapeTree.toStaticHashString()); + inner.hset("dump", key, actionSet.toString()); + inner.hset(key, "actionTree", actionTree.toStaticHashString()); + inner.hset(key, "targetTree", targetTree.toStaticHashString()); + inner.hset(key, "shapeTree", shapeTree.toStaticHashString()); } // File f = new File(root+"dumps/"+astNodeType+"/"+String.valueOf(size)+"/"); // f.mkdirs(); @@ -83,15 +83,19 @@ public class EDiffHunkParser extends EDiffParser { // oos.close(); } catch (Exception e) { - logger.error("error",e); + logger.error("error", e); // e.printStackTrace(); } hunkSet++; } + try (Jedis inner = innerPool.getResource()) { + inner.hset("diffEntry", pj + "_" + diffentryFile.getName(), "1"); + } } } + } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffMessage.java similarity index 96% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffMessage.java index e8bbbb0..8d57ec1 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffMessage.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; import edu.lu.uni.serval.fixminer.akka.BaseMessage; import redis.clients.jedis.JedisPool; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffParser.java similarity index 96% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffParser.java index bb418c1..70bd7a5 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffParser.java @@ -1,10 +1,11 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.gen.srcml.GumTreeCComparer; import edu.lu.uni.serval.gumtree.GumTreeComparer; +import edu.lu.uni.serval.utils.ListSorter; import redis.clients.jedis.JedisPool; import java.io.File; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffWorker.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffWorker.java index 54360a7..9882db1 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/EDiffWorker.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; import akka.actor.Props; import akka.actor.UntypedActor; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalActionSet.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalActionSet.java index a338d64..aaa87d3 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalActionSet.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.tree.ITree; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalRegrouper.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalRegrouper.java index 5587f9b..dd3c1fd 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalRegrouper.java @@ -1,8 +1,9 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.tree.ITree; import edu.lu.uni.serval.utils.ASTNodeMap; +import edu.lu.uni.serval.utils.ListSorter; import java.util.ArrayList; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouperForC.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalRegrouperForC.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouperForC.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalRegrouperForC.java index e36d6bb..8e0269a 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouperForC.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/HierarchicalRegrouperForC.java @@ -1,17 +1,15 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.gen.srcml.NodeMap_new; -import com.github.gumtreediff.io.CNodeMap; import com.github.gumtreediff.tree.ITree; -import edu.lu.uni.serval.gumtree.GumTreeComparer; +import edu.lu.uni.serval.utils.ListSorter; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; /** * Regroup GumTree results to a hierarchical construction. diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/HunkParserTest.java similarity index 95% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/HunkParserTest.java index a41f578..1f60347 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/HunkParserTest.java @@ -1,30 +1,19 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; -import com.github.gumtreediff.gen.srcml.NodeMap_new; import com.github.gumtreediff.tree.ITree; -import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser; -import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser; import edu.lu.uni.serval.utils.ClusterToPattern; import edu.lu.uni.serval.utils.EDiffHelper; import edu.lu.uni.serval.utils.PoolBuilder; import org.apache.commons.io.FileUtils; import org.javatuples.Pair; -import org.junit.Assert; import org.junit.Test; -import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import java.io.File; import java.io.IOException; -import java.lang.instrument.Instrumentation; -import java.nio.file.Files; -import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; public class HunkParserTest { @@ -246,7 +235,7 @@ public class HunkParserTest { String keyName = "if-3"; String i = "2"; String j = "21"; - HashMap filenames = AkkaTreeParser.filenames(outerPool); +// HashMap filenames = AkkaTreeParser.filenames(outerPool); // oldPair = EDiffHelper.getActions(keyName, i, outerPool, filenames); // newPair = EDiffHelper.getActions(keyName, j, outerPool, filenames); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/MessageFile.java similarity index 73% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/MessageFile.java index b51e597..e5a6597 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/MessageFile.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; import java.io.File; @@ -8,14 +8,22 @@ public class MessageFile { private File prevFile; private File diffEntryFile; private File positionFile; + + + + private String project; - public MessageFile(File revFile, File prevFile, File diffEntryFile) { + public MessageFile(File revFile, File prevFile, File diffEntryFile,String project) { super(); this.revFile = revFile; this.prevFile = prevFile; this.diffEntryFile = diffEntryFile; + this.project = project; } + public String getProject() { return project;} + + public void setProject(String project) { this.project = project;} public File getRevFile() { return revFile; } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/Parser.java similarity index 96% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/Parser.java index 91c68e6..b871882 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/Parser.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; import redis.clients.jedis.JedisPool; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/ParserInterface.java similarity index 88% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/ParserInterface.java index aac3e0f..e373440 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/ParserInterface.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; public interface ParserInterface { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/RunnableParser.java similarity index 67% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/RunnableParser.java index d0ee8ab..2890219 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/RunnableParser.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; import redis.clients.jedis.JedisPool; @@ -8,33 +8,33 @@ public class RunnableParser implements Runnable { private File prevFile; private File revFile; - private File diffentryFile; + private File diffEntryFile; private Parser parser; private String project; private JedisPool pool; private String srcMLPath; private String rootType; - public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) { + public RunnableParser(File prevFile, File revFile, File diffEntryFile, Parser parser) { this.prevFile = prevFile; this.revFile = revFile; - this.diffentryFile = diffentryFile; + this.diffEntryFile = diffEntryFile; this.parser = parser; } - public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool) { + public RunnableParser(File prevFile, File revFile, File diffEntryFile, Parser parser, String project, JedisPool innerPool) { this.prevFile = prevFile; this.revFile = revFile; - this.diffentryFile = diffentryFile; + this.diffEntryFile = diffEntryFile; this.parser = parser; this.project = project; this.pool = innerPool; } - public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser, String project, JedisPool innerPool,String srcMLPath,String rootType) { + public RunnableParser(File prevFile, File revFile, File diffEntryFile, Parser parser, String project, JedisPool innerPool,String srcMLPath,String rootType) { this.prevFile = prevFile; this.revFile = revFile; - this.diffentryFile = diffentryFile; + this.diffEntryFile = diffEntryFile; this.parser = parser; this.project = project; this.pool = innerPool; @@ -44,6 +44,6 @@ public class RunnableParser implements Runnable { @Override public void run() { - parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,pool,srcMLPath,rootType); + parser.parseFixPatterns(prevFile, revFile, diffEntryFile,project,pool,srcMLPath,rootType); } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/ediff/TestInputCases.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/TestInputCases.java new file mode 100644 index 0000000..cdae265 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/TestInputCases.java @@ -0,0 +1,50 @@ +package edu.lu.uni.serval.fixminer.ediff; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +public class TestInputCases { + + + @Test + public void test_189_B_17295034_17295064() throws IOException { + //cpython_b3a601_63d5c1_Objects#unicodeobject.c + //linux_659d8c_fd2a50a_tools#perf#builtin-kmem.c + //openssl_6a14fe7_0ff368_crypto#LPdir_win.c + //linux_d1dc69_cc2115c_sound#soc#codecs#wm8776.c + //linux_ff244c6_a0917e0_drivers#net#tun.c . + List hierarchicalActionSets = getHierarchicalActionSets("linux_cf36a65_fa6ce9_drivers#video#console#vgacon.c"); + Assert.assertEquals(hierarchicalActionSets.size(),1); + Assert.assertEquals(hierarchicalActionSets.get(0).toString(),""); + + } + + + + public List getHierarchicalActionSets(String s) throws IOException { + Properties appProps = new Properties(); + appProps.load(new FileInputStream("src/main/resource/app.properties")); + String srcMLPath = appProps.getProperty("srcMLPath", "FORKJOIN"); + String root = appProps.getProperty("inputPath"); + String project = s.split("_")[0]; + root = root + "/"+project+"/"; + String filename = s.replace(project+"_",""); + + File revFile = new File(root + "revFiles/" + filename); + File prevFile = new File(root + "prevFiles/prev_" + filename); + + EDiffHunkParser parser = new EDiffHunkParser(); + + + List hierarchicalActionSets = parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath); + return hierarchicalActionSets; + } + + +} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestIntroClass.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/TestIntroClass.java similarity index 99% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestIntroClass.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/TestIntroClass.java index 7ce7832..119f5c1 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestIntroClass.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/TestIntroClass.java @@ -1,7 +1,5 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; -import com.github.gumtreediff.tree.ITree; -import edu.lu.uni.serval.utils.EDiffHelper; import org.apache.commons.lang3.StringUtils; import org.junit.Assert; import org.junit.Test; @@ -13,7 +11,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestPredefinedCases.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/TestPredefinedCases.java similarity index 93% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestPredefinedCases.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/TestPredefinedCases.java index caed26b..1190f5e 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestPredefinedCases.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/TestPredefinedCases.java @@ -1,23 +1,12 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; + -import com.github.gumtreediff.tree.ITree; -import edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser; -import edu.lu.uni.serval.utils.ClusterToPattern; -import edu.lu.uni.serval.utils.EDiffHelper; -import edu.lu.uni.serval.utils.PoolBuilder; -import org.apache.commons.io.FileUtils; -import org.javatuples.Pair; import org.junit.Assert; import org.junit.Test; -import redis.clients.jedis.JedisPool; import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Properties; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestRealCases.java b/src/main/java/edu/lu/uni/serval/fixminer/ediff/TestRealCases.java similarity index 99% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestRealCases.java rename to src/main/java/edu/lu/uni/serval/fixminer/ediff/TestRealCases.java index c1c726c..649e711 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestRealCases.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/ediff/TestRealCases.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.fixminer.ediff; import com.github.gumtreediff.tree.ITree; import edu.lu.uni.serval.utils.EDiffHelper; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/CompareTrees.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/CompareTrees.java new file mode 100755 index 0000000..480e3d3 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/CompareTrees.java @@ -0,0 +1,230 @@ +package edu.lu.uni.serval.fixminer.jobs; + +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.PoolBuilder; +import me.tongfei.progressbar.ProgressBar; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + + +/** + * Created by anilkoyuncu on 03/04/2018. + */ +public class CompareTrees { + + private static Logger log = LoggerFactory.getLogger(CompareTrees.class); + + + public static void main(String redisPath, String portDumps, String dumpsName, String job,String numOfWorkers,String host) throws Exception { + + // shape /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis ALLdumps-gumInput.rdb clusterl0-gumInputALL.rdb /Users/anil.koyuncu/projects/test/fixminer-core/python/data/richEditScript + +// String portInner = "6380"; + String port = portDumps; //"6399"; + CallShell cs = new CallShell(); + String cmd = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s"; + cmd = String.format(cmd, redisPath,dumpsName,Integer.valueOf(port)); + log.info(cmd); + cs.runShell(cmd, port); + +// String cmdInner = "bash "+redisPath + "/" + "startServer.sh" +" %s %s %s"; +// cmdInner = String.format(cmdInner, redisPath,compareDBName,Integer.valueOf(portInner)); +// log.info(cmdInner); +// cs.runShell(cmdInner, portInner); + +// String numOfWorkers = "100000000";//args[4]; +// String host = "localhost";//args[5]; +// -Djava.util.concurrent.ForkJoinPool.common.parallelism=256 + +// final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000); + + final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(port),20000000); + +// List listOfPairs = AkkaTreeParser.getMessages(innerPool,Integer.valueOf(numOfWorkers)); + HashMap filenames = getFilenames(outerPool); +// List listOfPairs = AkkaTreeParser.files2compare(outerPool); + + + ArrayList samePairs = new ArrayList<>(); + ArrayList errorPairs = new ArrayList<>(); + + Integer numberOfWorkers = Integer.valueOf(numOfWorkers); + final ExecutorService executor = Executors.newWorkStealingPool(numberOfWorkers); + ArrayList> results = new ArrayList>(); + for (int i = 1; i future = executor.submit(new RunnableCompare(job, errorPairs, filenames, outerPool, i)); + results.add(future); + } + for (Future future : ProgressBar.wrap(results, "Comparing")){ +// for (Future future:results){ + try { + // wait for task to complete + future.get(); + + } catch (InterruptedException e) { + + e.printStackTrace(); + } catch (ExecutionException e) { + + e.printStackTrace(); + } +// finally { +// executor.shutdownNow(); +// } + } + executor.shutdownNow(); + + + + + log.info("End process"); + } + + + public static class RunnableCompare implements Runnable { + String job; + ArrayList errorPairs; + HashMap filenames; + JedisPool outerPool; + Integer threadID; + + public RunnableCompare(String treeType,ArrayList errorPairs, HashMap filenames,JedisPool outerPool,Integer threadID) { + this.job = treeType; + this.errorPairs = errorPairs; + this.filenames = filenames; + this.outerPool = outerPool; + this.threadID = threadID; + } + + @Override + public void run() { +// int dbsize = 1; + boolean stop = true; + while(stop) { +// try (Jedis outer = outerPool.getResource()) { +// dbsize = Math.toIntExact(outer.scard("compare")); +// } +// if (dbsize != 0){ + stop = newCoreCompare(job, errorPairs, filenames, outerPool); +// } + } + log.info("Completed worker {}",threadID); + } + } + + + public static boolean newCoreCompare( String treeType,ArrayList errorPairs, HashMap filenames,JedisPool outerPool ) { + + String pairName; + try (Jedis outer = outerPool.getResource()) { + pairName = outer.spop("compare"); + } + + String matchKey = null; + try { + + String[] split = pairName.split("/"); + + + String i = split[1]; + String j = split[2]; + String keyName = split[0]; + matchKey = keyName + "/" + (String.valueOf(i)) + "/" + String.valueOf(j); + + switch (treeType) { + case "single": + + if (matchKey == null){ + return false; + } + Map oldTreeString = EDiffHelper.getTreeString(keyName, i, outerPool, filenames); + Map newTreeString = EDiffHelper.getTreeString(keyName, j, outerPool, filenames); + + String oldShapeTree =oldTreeString.get("shapeTree"); + String newShapeTree =newTreeString.get("shapeTree"); + + String oldActionTree = oldTreeString.get("actionTree"); + String newActionTree = newTreeString.get("actionTree"); + + String oldTargetTree = oldTreeString.get("targetTree"); + String newTargetTree = newTreeString.get("targetTree"); + + + if (oldShapeTree.equals(newShapeTree)) { + if (oldActionTree.equals(newActionTree)) { + if (oldTargetTree.equals(newTargetTree)) { +// samePairs.add(matchKey); + try (Jedis jedis = outerPool.getResource()) { +//// jedis.del(matchKey); + jedis.select(2); + jedis.set(matchKey, "1"); + } + } + } + } + return true; + default: + return true; +// break; + } + }catch (Exception e) { + errorPairs.add(matchKey); + if (pairName == null) + return false; + log.debug("{} not comparable", pairName); + } + + + return true; + } + + + + public static HashMap getFilenames(JedisPool innerPool){ + + + HashMap fileMap =new HashMap(); + + try (Jedis inner = innerPool.getResource()) { + while (!inner.ping().equals("PONG")){ + log.info("wait"); + } + + inner.select(1); + Map filenames = inner.hgetAll("filenames"); + + + for (Map.Entry stringStringEntry : filenames.entrySet().stream().collect(Collectors.toList())) { + fileMap.put(stringStringEntry.getKey(),stringStringEntry.getValue()); + } + + + + + + } + +// log.info("Getting results %d",fileMap.s); + return fileMap; + } + + + + +} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java index 34a3660..d07dd45 100755 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java @@ -1,22 +1,21 @@ package edu.lu.uni.serval.fixminer.jobs; -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import edu.lu.uni.serval.fixminer.akka.ediff.*; +import edu.lu.uni.serval.fixminer.ediff.EDiffHunkParser; +import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.fixminer.ediff.MessageFile; import edu.lu.uni.serval.utils.CallShell; import edu.lu.uni.serval.utils.PoolBuilder; -import org.apache.commons.io.FileUtils; +import me.tongfei.progressbar.ProgressBar; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -27,7 +26,7 @@ public class EnhancedASTDiff { private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class); - public static void main(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType,String hunkLimit) throws Exception { + public static void main(String inputPath, String project, String portInner, String dbDir, String chunkName,String srcMLPath,String parameter,String hunkLimit,String[] projectList,String patchSize) throws Exception { String parameters = String.format("\nInput path %s",inputPath); @@ -35,34 +34,19 @@ public class EnhancedASTDiff { CallShell cs = new CallShell(); String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; -// if (rootType == null){ + cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner)); -// }else{ -// cmd = String.format(cmd, dbDir,rootType+chunkName,Integer.valueOf(portInner)); -// } cs.runShell(cmd, portInner); JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000); - if (rootType == "add"){ - try (Jedis inner = innerPool.getResource()) { - inner.select(2); - inner.flushDB(); - inner.select(1); - inner.flushDB(); - inner.select(0); - inner.del("compare"); - - } - + List> allPredicates = new ArrayList>(); + for (String s : projectList) { + Predicate predicate = x->x.getName().endsWith(s); + allPredicates.add(predicate); } - Predicate predicate1 = x -> x.getName().endsWith("libtiff"); - Predicate predicate2 = x -> x.getName().endsWith("php-src"); - Predicate predicate3 = x -> x.getName().endsWith("cpython"); - Predicate predicate4 = x -> x.getName().endsWith("wireshark"); - Predicate predicate5 = x -> x.getName().endsWith("gzip"); File folder = new File(inputPath); File[] listOfFiles = folder.listFiles(); Stream stream = Arrays.stream(listOfFiles); @@ -70,15 +54,14 @@ public class EnhancedASTDiff { .filter(x -> !x.getName().startsWith(".")) .filter(x -> !x.getName().startsWith("cocci")) .filter(x -> !x.getName().endsWith(".index")) -// .filter(x -> x.getName().endsWith("libtiff")) - .filter(predicate1.or(predicate2).or(predicate3).or(predicate4).or(predicate5)) + .filter(allPredicates.stream().reduce(x->false, Predicate::or)) .collect(Collectors.toList()); List allMessageFiles = new ArrayList<>(); for (File target : folders) { - List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" + List msgFiles = getMessageFiles(target.toString() + "/",project,patchSize); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" // msgFiles = msgFiles.subList(0,3000); if (msgFiles == null) @@ -88,64 +71,31 @@ public class EnhancedASTDiff { } - switch (parallelism){ - case "AKKA": - ActorSystem system = null; - ActorRef parsingActor = null; + Map diffEntry; + try (Jedis inner = innerPool.getResource()) { + diffEntry = inner.hgetAll("diffEntry"); - final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,innerPool,srcMLPath,hunkLimit); - try { - log.info("Akka begins..."); - log.info("{} files to process ...", allMessageFiles.size()); - system = ActorSystem.create("Mining-FixPattern-System"); - - parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor"); - parsingActor.tell(msg, ActorRef.noSender()); - } catch (Exception e) { - system.shutdown(); - e.printStackTrace(); - }finally { - system.awaitTermination(); -// system.shutdown(); - } - break; - case "FORKJOIN": - int counter = new Object() { - int counter = 0; - - { - allMessageFiles.stream(). - parallel(). - peek(x -> counter++). - forEach(m -> - { - EDiffHunkParser parser = new EDiffHunkParser(); - parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool,srcMLPath,hunkLimit); - if (counter % 10 == 0) { - log.info("Finalized parsing " + counter + " files... remaining " + (allMessageFiles.size() - counter)); - } - } - ); - } - }.counter; - log.info("Finished parsing {} files",counter); - break; - - - default: - log.error("Unknown parallelism {}", parallelism); - break; } - - - - + log.info("{} files to process ...", allMessageFiles.size()); + if (diffEntry != null) { + log.info("{} files already process ...", diffEntry.size()); + allMessageFiles = allMessageFiles.stream().filter(f -> !diffEntry.containsKey(f.getProject() + "_" + f.getDiffEntryFile().getName())).collect(Collectors.toList()); + log.info("{} files to process ...", allMessageFiles.size()); + } + ProgressBar.wrap(allMessageFiles.stream(). + parallel(),"Task"). + forEach(m -> + { + EDiffHunkParser parser = new EDiffHunkParser(); + parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,innerPool,srcMLPath,hunkLimit); + } + ); } - private static List getMessageFiles(String gumTreeInput) { + private static List getMessageFiles(String gumTreeInput,String datasetName,String patchSize) { String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile File revFilesPath = new File(inputPath + "revFiles/"); log.info(revFilesPath.getPath()); @@ -167,12 +117,18 @@ public class EnhancedASTDiff { int count = 0; while (matcher.find()) count++; - if(count>51) + if(count>= Integer.valueOf(patchSize)) // if(count>201) continue; // if(FileHelper.readFile(diffentryFile).split("@@\\s\\-\\d+,*\\d*\\s\\+\\d+,*\\d*\\s@@").length > 2) // continue; - MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); + +// String datasetName = project; + String[] split1 = diffentryFile.getParent().split(datasetName); + String root = split1[0]; + String pj = split1[1].split("/")[1]; + + MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile,pj); msgFiles.add(msgFile); @@ -184,69 +140,6 @@ public class EnhancedASTDiff { } } - public static void load(String inputPath, String numOfWorkers, String project, String eDiffTimeout, String parallelism, String portInner, String dbDir, String chunkName,String srcMLPath,String rootType) throws Exception { - - - String parameters = String.format("\nInput path %s",inputPath); - log.info(parameters); - - CallShell cs = new CallShell(); - String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; - cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner)); -// if (rootType == null){ -// cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner)); -// }else{ -// cmd = String.format(cmd, dbDir,rootType+chunkName,Integer.valueOf(portInner)); -// } - - cs.runShell(cmd, portInner); - - JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000); - try (Jedis inner = innerPool.getResource()) { - inner.flushAll(); - } - File folder = new File(new File(inputPath).getParent() + "/dumps/" + rootType); - File[] listOfFiles = folder.listFiles(); - Stream stream = Arrays.stream(listOfFiles); - List folders = stream - .filter(x -> !x.getName().startsWith(".")) - .collect(Collectors.toList()); - List allMessageFiles = new ArrayList<>(); - - for (File target : folders) { - if(target.getName().startsWith(".")) - continue; - List files = Arrays.asList(target.listFiles()); - if (files.size() > 1){ - allMessageFiles.addAll(files); - } - } - - log.info("Message size: "+allMessageFiles.size()); - - allMessageFiles.stream(). - parallel(). - forEach(x-> loadCore(x,innerPool)); - - } - - public static void loadCore(File file2load, JedisPool innerPool){ - try (Jedis inner = innerPool.getResource()) { - -// byte[] dump = Files.readAllBytes(Paths.get(file2load.getPath())); - byte[] dump = FileUtils.readFileToByteArray(file2load); -// HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.kryoDeseerialize(dump); - String key = file2load.getPath().split("/dumps/")[1]; - inner.hset("dump".getBytes(),key.getBytes(),dump); -// actionSet.toString(); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - - } - } diff --git a/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java b/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java index 2ddee46..e2d5fc7 100755 --- a/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java +++ b/src/main/java/edu/lu/uni/serval/utils/ClusterToPattern.java @@ -1,6 +1,5 @@ package edu.lu.uni.serval.utils; -import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java index 40e8dc4..fe09bc2 100755 --- a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java +++ b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java @@ -5,10 +5,8 @@ import com.github.gumtreediff.gen.srcml.NodeMap_new; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; import com.github.gumtreediff.tree.TreeUtils; -import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; -import edu.lu.uni.serval.fixminer.akka.ediff.KryoContext; +import edu.lu.uni.serval.fixminer.ediff.HierarchicalActionSet; import org.apache.commons.lang3.SerializationUtils; -import org.javatuples.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/FileHelper.java b/src/main/java/edu/lu/uni/serval/utils/FileHelper.java similarity index 99% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/FileHelper.java rename to src/main/java/edu/lu/uni/serval/utils/FileHelper.java index 94967cc..5d4bb40 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/FileHelper.java +++ b/src/main/java/edu/lu/uni/serval/utils/FileHelper.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.utils; import java.io.BufferedInputStream; import java.io.BufferedWriter; import java.io.File; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ListSorter.java b/src/main/java/edu/lu/uni/serval/utils/ListSorter.java similarity index 95% rename from src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ListSorter.java rename to src/main/java/edu/lu/uni/serval/utils/ListSorter.java index 5f1543f..ac69440 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ListSorter.java +++ b/src/main/java/edu/lu/uni/serval/utils/ListSorter.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.akka.ediff; +package edu.lu.uni.serval.utils; import java.util.ArrayList; import java.util.Collections; diff --git a/src/main/resource/app.properties b/src/main/resource/app.properties index 3970991..06c5cb0 100755 --- a/src/main/resource/app.properties +++ b/src/main/resource/app.properties @@ -1,20 +1,23 @@ -pjName = gumInput -portInner = 6380 +pjName = patches portDumps = 6399 -parallelism = AKKA numOfWorkers = 14 hostname = localhost hunkLimit = 10 +patchSize = 50 + +projectList = libtiff,php-src,cpython,wireshark,gzip,gmp,lighttpd1.4,lighttpd2 #inputPath = /Users/anilkoyuncu/projects/gumInputLinux -inputPath = /Users/anil.koyuncu/projects/fixminer/fixminer-data/gumInputLinux +#inputPath = /Users/anil.koyuncu/projects/fixminer/fixminer-data/gumInputLinux +inputPath = /Users/anil.koyuncu/projects/test/fixminer-data/patches #redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis -redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis -srcMLPath= /Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml +#redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis +redisPath = /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis +#srcMLPath= /Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml +srcMLPath= /Users/anil.koyuncu/projects/test/srcML/bin/srcml #srcMLPath= /usr/local/bin/srcml actionType = ALL -eDiffTimeout = 900 #ENHANCEDASTDIFF,CACHE,LEVEL1,LEVEL2,LEVEL3