diff --git a/pom.xml b/pom.xml index 1378d5b..09f7d2d 100644 --- a/pom.xml +++ b/pom.xml @@ -108,8 +108,8 @@ - edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader - + edu.lu.uni.serval.FixPatternParser.cluster.CalculatePairs + diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java new file mode 100644 index 0000000..1e2e76e --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java @@ -0,0 +1,327 @@ +package edu.lu.uni.serval.FixPatternParser.cluster; + +import com.github.gumtreediff.tree.ITree; +import com.github.gumtreediff.tree.Tree; +import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.*; + +import java.io.*; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.Executors; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** + * Created by anilkoyuncu on 19/03/2018. + */ +public class AkkaTreeLoader { + + private static class StreamGobbler implements Runnable { + private InputStream inputStream; + private Consumer consumer; + + public StreamGobbler(InputStream inputStream, Consumer consumer) { + this.inputStream = inputStream; + this.consumer = consumer; + } + + @Override + public void run() { + new BufferedReader(new InputStreamReader(inputStream)).lines() + .forEach(consumer); + } + } + + private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class); + + + public static void loadRedis(String cmd,String serverWait){ + Process process; + + try { +// String comd = String.format(cmd, f.getAbsoluteFile()); + process = Runtime.getRuntime() + + .exec(cmd); + + + StreamGobbler streamGobbler = + new StreamGobbler(process.getInputStream(), System.out::println); + Executors.newSingleThreadExecutor().submit(streamGobbler); +// int exitCode = process.waitFor(); +// assert exitCode == 0; + Thread.sleep(Integer.valueOf(serverWait)); + + } catch (IOException e) { + e.printStackTrace(); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + log.info("Load done"); + } + + public static void main(String[] args) { + + + String inputPath; +// String outputPath; + String port; + String portInner; +// String pairsCSVPath; + String importScript; +// String pairsCompletedPath; + String serverWait; +// String option; + String dbDir; + String chunkName; + String numOfWorkers; + if (args.length > 0) { + inputPath = args[0]; + portInner = args[1]; + serverWait = args[2]; +// option = args[4]; + chunkName = args[3]; + numOfWorkers = args[4]; + dbDir = args[5]; + port = args[6]; +// pairsCSVPath = args[3]; +// importScript = args[4]; +// pairsCompletedPath = args[3]; + } else { + inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; +// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/"; + port = "6399"; + portInner = "6380"; + serverWait = "10000"; +// option = "COMP"; +// pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test"; +// importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh"; +// pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed"; + chunkName ="chunk3.rdb"; + dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; + numOfWorkers = "1"; + + } + String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \nnumOfWorks %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,numOfWorkers,dbDir); + log.info(parameters); + +// if (option.equals("CALC")) { +// calculatePairs(inputPath, port); +// log.info("Calculate pairs done"); +// }else { + String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; + cmd = String.format(cmd, dbDir,"dumps.rdb",Integer.valueOf(port)); + loadRedis(cmd,serverWait); + + String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; + cmd = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner)); + loadRedis(cmd,serverWait); + + JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); + JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000); + + + + comparePairs(inputPath, innerPool,outerPool, serverWait,chunkName,dbDir,numOfWorkers); + + String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; + stopServer = String.format(stopServer,Integer.valueOf(portInner)); + loadRedis(stopServer,serverWait); +// } + + + } + + + public static void comparePairs(String inputPath, JedisPool innerPool,JedisPool outerPool,String serverWait,String chunkName, String dbDir,String numOfWorkers){ + + + ScanResult scan; + try (Jedis inner = innerPool.getResource()) { + while (inner.ping()== "PONG"){ + log.info("wait"); + } + + ScanParams sc = new ScanParams(); + //150000000 + sc.count(150000000); + sc.match("pair_*"); + + scan = inner.scan("0", sc); + int size = scan.getResult().size(); + log.info("Scanning " + String.valueOf(size)); + } + List result = scan.getResult(); + + +// ActorSystem system = null; +// ActorRef parsingActor = null; +// final WorkMessage msg = new WorkMessage(0, result,innerPort,inputPath,dbDir,serverWait); +// try { +// +// log.info("Akka begins..."); +// system = ActorSystem.create("Tree-System"); +// parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers),dbDir,innerPort,serverWait), "tree-actor"); +// parsingActor.tell(msg, ActorRef.noSender()); +// } catch (Exception e) { +// system.shutdown(); +// e.printStackTrace(); +// } +// greeter.tell(); + result + .parallelStream() + .forEach(m -> + { + Compare compare = new Compare(); + compare.coreCompare(m, innerPool, outerPool); + ; + } + ); + + + + +// } + + + + + } + + /** Read the object from Base64 string. */ + private static Object fromString( String s ) throws IOException , + ClassNotFoundException { + byte [] data = Base64.getDecoder().decode( s ); + ObjectInputStream ois = new ObjectInputStream( + new ByteArrayInputStream( data ) ); + Object o = ois.readObject(); + ois.close(); + return o; + } + public static ITree getSimpliedTree(String fn,JedisPool outerPool) { + + ITree tree = null; + Jedis inner = null; + try { + inner = outerPool.getResource(); + String s = inner.get(fn.substring(1)); + HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s); + + ITree parent = null; + ITree children =null; + tree = getASTTree(actionSet, parent, children); + tree.setParent(null); + + } catch (IOException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + }finally { + if (inner != null) { + inner.close(); + } + } + return tree; + + } + + public static List getKeysByValue(Map map, E value) { + return map.entrySet() + .stream() + .filter(entry -> Objects.equals(entry.getValue(), value)) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } + + public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children){ + + int newType = 0; + + String astNodeType = actionSet.getAstNodeType(); + List keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType); + + if(keysByValue.size() != 1){ + log.error("Birden cok astnodemapmapping"); + } + newType = keysByValue.get(0); + if(actionSet.getParent() == null){ + //root + + parent = new Tree(newType,""); + }else{ + children = new Tree(newType,""); + parent.addChild(children); + } + List subActions = actionSet.getSubActions(); + if (subActions.size() != 0){ + for (HierarchicalActionSet subAction : subActions) { + + if(actionSet.getParent() == null){ + children = parent; + } + getASTTree(subAction,children,null); + + } + + + } + return parent; + } + + + static final JedisPoolConfig poolConfig = buildPoolConfig(); + + + private static JedisPoolConfig buildPoolConfig() { + final JedisPoolConfig poolConfig = new JedisPoolConfig(); + poolConfig.setMaxTotal(128); + poolConfig.setMaxIdle(128); + poolConfig.setMinIdle(16); + poolConfig.setTestOnBorrow(false); + poolConfig.setTestOnReturn(false); + poolConfig.setTestWhileIdle(true); + poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); + poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); +// poolConfig.setNumTestsPerEvictionRun(3); + poolConfig.setBlockWhenExhausted(true); + + return poolConfig; + } + + private static List getMessageFiles(String gumTreeInput) { + String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile + File revFilesPath = new File(inputPath + "revFiles/"); + File[] revFiles = revFilesPath.listFiles(); // project folders + List msgFiles = new ArrayList<>(); + if (revFiles.length >= 0) { + for (File revFile : revFiles) { +// if (revFile.getName().endsWith(".java")) { + String fileName = revFile.getName(); + File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file + fileName = fileName.replace(".java", ".txt"); + File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file + File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file + edu.lu.uni.serval.MultipleThreadsParser.MessageFile msgFile = new edu.lu.uni.serval.MultipleThreadsParser.MessageFile(revFile, prevFile, diffentryFile); + msgFile.setPositionFile(positionFile); + msgFiles.add(msgFile); +// } + } + + return msgFiles; + } + else{ + return null; + } + } + + +} + + + + diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java new file mode 100644 index 0000000..8f8c558 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java @@ -0,0 +1,153 @@ +package edu.lu.uni.serval.FixPatternParser.cluster; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.ScanParams; +import redis.clients.jedis.ScanResult; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.List; + +import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; +import static edu.lu.uni.serval.FixPatternParser.cluster.TreeLoaderClusterL1.poolConfig; + +/** + * Created by anilkoyuncu on 05/04/2018. + */ +public class CalculatePairs { + private static Logger log = LoggerFactory.getLogger(CalculatePairs.class); + public static void main(String[] args) { + + + String inputPath; + String port; + String portInner; + String serverWait; + String dbDir; + String chunkName; + + if (args.length > 0) { + inputPath = args[0]; + portInner = args[1]; + serverWait = args[2]; + chunkName = args[3]; + + dbDir = args[5]; + port = args[6]; + + } else { + inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; + + port = "6399"; + portInner = "6380"; + serverWait = "10000"; + + chunkName ="chunk"; + dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; + + + } + String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,dbDir); + log.info(parameters); + + + String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; + cmd = String.format(cmd, dbDir,"dumps.rdb",Integer.valueOf(port)); + loadRedis(cmd,serverWait); + + String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; + cmd = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner)); + loadRedis(cmd,serverWait); + + JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); + JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000); + + + ScanResult scan; + try (Jedis outer = outerPool.getResource()) { + while (outer.ping()== "PONG"){ + log.info("wait"); + } + + ScanParams sc = new ScanParams(); + //150000000 + sc.count(150000000); + sc.match("*"); + + scan = outer.scan("0", sc); + int size = scan.getResult().size(); + log.info("Scanning " + String.valueOf(size)); + } + List result = scan.getResult(); + + int fileCounter = 0; + int pairCounter = 0; + for (int i = 0; i < result.size(); i++) { + for (int j = i + 1; j < result.size(); j++) { + Jedis jedis = null; + String key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j); + try { + jedis = innerPool.getResource(); + key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j); +// String value = treesFileNames.get(i).split("GumTreeOutput2")[1] +","+treesFileNames.get(j).split("GumTreeOutput2")[1]; +// jedis.set(key,value); + + jedis.hset(key, "0", result.get(i)); + jedis.hset(key, "1", result.get(j)); + pairCounter ++; + + //10000000 + if (pairCounter % 10000000 == 0) { + + File dbPath = new File(dbDir + "/" + chunkName); + File savePath = new File(dbDir + "/" + "chunk" + String.valueOf(fileCounter) + ".rdb"); + try { + jedis.save(); + log.info("saving key {} chunk {}",key,fileCounter); + while (jedis.ping() == "PONG") { + log.info("wait"); + } + + + Files.copy(dbPath.toPath(), savePath.toPath(), StandardCopyOption.REPLACE_EXISTING); + fileCounter++; + jedis.flushDB(); + while (jedis.ping() == "PONG") { + log.info("wait"); + } + + } catch (IOException e) { + + e.printStackTrace(); + } + + + } + }catch (Exception e) { + log.error(e.toString() + " {}", (key)); + + + }finally { + if (jedis != null) { + jedis.close(); + } + } + } + } + + +// comparePairs(inputPath, innerPool,outerPool, serverWait,chunkName,dbDir,numOfWorkers); + + String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; + stopServer = String.format(stopServer,Integer.valueOf(portInner)); + loadRedis(stopServer,serverWait); +// } + + + } +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/Compare.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/Compare.java similarity index 92% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/Compare.java rename to src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/Compare.java index 813c982..76350e0 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/Compare.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/Compare.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser.violations; +package edu.lu.uni.serval.FixPatternParser.cluster; import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.model.Action; @@ -9,13 +9,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; -import java.time.Duration; import java.util.List; import java.util.Map; -import static edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader.getSimpliedTree; +import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.getSimpliedTree; /** * Created by anilkoyuncu on 03/04/2018. diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/StoreFile.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java similarity index 58% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/StoreFile.java rename to src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java index 588b099..a8ff74a 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/StoreFile.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java @@ -1,10 +1,5 @@ -package edu.lu.uni.serval.FixPatternParser.violations; +package edu.lu.uni.serval.FixPatternParser.cluster; -import com.github.gumtreediff.actions.ActionGenerator; -import com.github.gumtreediff.actions.model.Action; -import com.github.gumtreediff.matchers.Matcher; -import com.github.gumtreediff.matchers.Matchers; -import com.github.gumtreediff.tree.ITree; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,14 +9,14 @@ import redis.clients.jedis.JedisPoolConfig; import redis.clients.jedis.ScanResult; import java.io.*; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; -import static edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader.loadRedis; -import static edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader.getSimpliedTree; +import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; /** * Created by anilkoyuncu on 03/04/2018. @@ -156,82 +151,5 @@ public class StoreFile { return Base64.getEncoder().encodeToString(baos.toByteArray()); } - public void storeBinary(String name , String inputPath, String innerPort) { - JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000); - Map resultMap; - try (Jedis jedis = pool.getResource()) { - resultMap = jedis.hgetAll(name); - } - String[] split = name.split("_"); - - String i = split[1]; - String j = split[2]; - String firstValue = resultMap.get("0"); - String secondValue = resultMap.get("1"); - - String[] firstValueSplit = firstValue.split("GumTreeOutput2"); - String[] secondValueSplit = secondValue.split("GumTreeOutput2"); - - - - if (firstValueSplit.length == 1) { - firstValue = inputPath + firstValueSplit[0]; - } else { - firstValue = inputPath + firstValueSplit[1]; - } - - if (secondValueSplit.length == 1) { - secondValue = inputPath + secondValueSplit[0]; - } else { - secondValue = inputPath + secondValueSplit[1]; - } - - try { - ITree oldTree = getSimpliedTree(firstValue); - - ITree newTree = getSimpliedTree(secondValue); - - Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); - m.match(); - - - ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); - ag.generate(); - List actions = ag.getActions(); - - double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree); - String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1); - double diceSimilarity1 = m.diceSimilarity(oldTree, newTree); - String diceSimilarity = String.format("%1.2f", diceSimilarity1); - double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree); - String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); - - String editDistance = String.valueOf(actions.size()); - - String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; - - - if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) - || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { - String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); - log.info(matchKey); - try (Jedis jedis = pool.getResource()) { - jedis.select(1); - jedis.set(matchKey, result); - } - } - - - try (Jedis jedis = pool.getResource()) { - jedis.del("pair_" + (String.valueOf(i)) + "_" + String.valueOf(j)); - } - - - } catch (Exception e) { - log.error(e.toString() + " {}", (name)); - - - } - } } diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java new file mode 100644 index 0000000..dd82e21 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java @@ -0,0 +1,767 @@ +package edu.lu.uni.serval.FixPatternParser.cluster; + +import com.github.gumtreediff.actions.ActionGenerator; +import com.github.gumtreediff.actions.model.*; +import com.github.gumtreediff.matchers.Matcher; +import com.github.gumtreediff.matchers.Matchers; +import com.github.gumtreediff.tree.ITree; +import edu.lu.uni.serval.gumtree.GumTreeComparer; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; +import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.ListSorter; +import org.javatuples.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.*; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.Executors; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; + +/** + * Created by anilkoyuncu on 19/03/2018. + */ +public class TreeLoaderClusterL1 { + + private static int resultType; + + + private static Logger log = LoggerFactory.getLogger(TreeLoaderClusterL1.class); + + public static void main(String[] args){ + + String inputPath; + String outputPath; + String port; + String portInner; + String serverWait; + if (args.length > 0) { + inputPath = args[0]; + outputPath = args[1]; + port = args[2]; + serverWait = args[3]; + portInner = args[4]; + + } else { +// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2/"; + inputPath = "/Users/anilkoyuncu/bugStudy/dataset/redis"; + outputPath = "/Users/anilkoyuncu/bugStudy/dataset/"; + port = "6379"; + portInner = "6380"; + serverWait = "10000"; + + + } + + String cmd = "bash "+inputPath + "/" + "startServer.sh" +" %s %s %s"; + cmd = String.format(cmd, inputPath,"level1.rdb",Integer.valueOf(port)); + loadRedis(cmd,serverWait); + + JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); + + + String level1Path = inputPath + "/level1"; + File chunks = new File(level1Path); + File[] listFiles = chunks.listFiles(); + Stream stream = Arrays.stream(listFiles); + List dbs = stream + .filter(x -> x.getName().endsWith(".rdb")) + .collect(Collectors.toList()); + for (File db : dbs) { + String cmdInner = "bash "+level1Path + "/" + "startServer.sh" +" %s %s %s"; + cmdInner = String.format(cmdInner, level1Path,db.getName(),Integer.valueOf(portInner)); + loadRedis(cmdInner,serverWait); + JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000); + + Jedis inner = null; + + try { + inner = innerPool.getResource(); + + inner.select(1); + ScanParams sc = new ScanParams(); + //150000000 + sc.count(150000000); + sc.match("match_*"); + + ScanResult scan; scan = inner.scan("0", sc); + int size = scan.getResult().size(); + log.info("Scanning " + String.valueOf(size)); + + List result = scan.getResult(); + + for (String key : result) { + String value = inner.get(key); + Jedis outer = null; + try { + outer = outerPool.getResource(); + outer.set(key,value); + } catch (Exception e) { + log.error(e.toString() + " {}", (key)); + + + }finally { + if (outer != null) { + outer.close(); + } + } + + + } + + + } catch (Exception e) { + log.error(e.toString() + " {}", (db.getName())); + + + }finally { + if (inner != null) { + inner.close(); + } + innerPool.close(); + } + + String stopServer = "bash "+level1Path + "/" + "stopServer.sh" +" %s"; + stopServer = String.format(stopServer,Integer.valueOf(portInner)); + loadRedis(stopServer,serverWait); + + } + + +// calculatePairsOfClusters(inputPath, outputPath); +// mainCompare(inputPath,port,pairsCSVPath,importScript); + // calculatePairs(inputPath, outputPath); +// processMessages(inputPath,outputPath); +// evaluateResults(inputPath,outputPath); + + } + + + public static void mainCompare(String inputPath,String port,String pairsCSVPath,String importScript) { + + String cmd; + cmd = "bash " + importScript +" %s"; + + JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); + + + File folder = new File(pairsCSVPath); + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List folders = stream + .filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); + + for (File f:folders){ + + if(f.getName().startsWith("cluster76")) { + + + try (Jedis jedis = jedisPool.getResource()) { + // do operations with jedis resource + ScanParams sc = new ScanParams(); + sc.count(150000000); + sc.match("pair_[0-9]*"); + + log.info("Scanning"); + ScanResult scan = jedis.scan("0", sc); + int size = scan.getResult().size(); + + if (size == 0) { +// loadRedis(cmd, f); + + scan = jedis.scan("0", sc); + size = scan.getResult().size(); + + } + log.info("Scanned " + String.valueOf(size)); + + + String clusterName = f.getName().replaceAll("[^0-9]+", ""); + + + //76 + + scan.getResult().parallelStream() + .forEach(m -> coreCompare(m, inputPath, jedisPool, clusterName)); + + + jedis.save(); + + } + } + + + } + + + + } + + + + + + public static Pair getTree(String firstValue){ + String gumTreeInput = "/Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/"; + String[] split2 = firstValue.split("/"); + String cluster = split2[1]; + + + File folder = new File("/Users/anilkoyuncu/bugStudy/code/python/cluster/"+cluster); + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List folders = stream + .filter(x -> !x.getName().startsWith(".") && x.getName().startsWith(split2[2])) + .collect(Collectors.toList()); + + + + String[] split1 = folders.get(0).getName().split(".txt_"); + String s = split1[0]; + String[] splitPJ = split1[1].split("_"); + String project = splitPJ[1]; + String actionSetPosition = splitPJ[0]; + + File prevFile = new File(gumTreeInput + project+ "/" + "prevFiles/prev_" + s + ".java");// previous file + File revFile = new File(gumTreeInput + project+ "/" + "revFiles/" + s + ".java");//rev file + + List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); + + HierarchicalActionSet actionSet = actionSets.get(Integer.valueOf(actionSetPosition)); +// for (HierarchicalActionSet actionSet : actionSets) { + + + + ITree actionTree= null; + ITree test2 = null; + getActionTree(actionSet); + ITree node = actionSet.getNode(); + List descendants = node.getDescendants(); + for (ITree descendant : descendants) { + if(descendant.getType() <= 100){ + descendant.setType(104); + } + } + node.setParent(null); + + + // } +// } + + Pair pair = new Pair<>(node,project); + return pair; + } + + + public static void getActionTree(HierarchicalActionSet actionSet){ + + + int newType = 0; + + Action action = actionSet.getAction(); + if (action instanceof Update){ + newType = 101; + }else if(action instanceof Insert){ + newType =100; + }else if(action instanceof Move){ + newType = 102; + }else if(action instanceof Delete){ + newType=103; + }else{ + new Exception("unknow action"); + } + actionSet.getNode().setType(newType); +// actionSet.getNode().setLabel(""); + List subActions = actionSet.getSubActions(); + if (subActions.size() != 0){ + for (HierarchicalActionSet subAction : subActions) { + getActionTree(subAction); + } + + + } + + } +// public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children){ +// +// int newType = 0; +// +// Action action = actionSet.getAction(); +// if (action instanceof Update){ +// newType = 101; +// }else if(action instanceof Insert){ +// newType =100; +// }else if(action instanceof Move){ +// newType = 102; +// }else if(action instanceof Delete){ +// newType=103; +// }else{ +// new Exception("unknow action"); +// } +// if(actionSet.getParent() == null){ +// //root +// +// parent = new Tree(newType,""); +// }else{ +// children = new Tree(newType,""); +// parent.addChild(children); +// } +// List subActions = actionSet.getSubActions(); +// if (subActions.size() != 0){ +// for (HierarchicalActionSet subAction : subActions) { +// +// if(actionSet.getParent() == null){ +// children = parent; +// } +// getActionTree(subAction,children,null); +// +// } +// +// +// } +// return parent; +// } + + + + private static void coreCompare(String name , String inputPath, JedisPool jedisPool,String clusterName) { + + + try (Jedis jedis = jedisPool.getResource()) { + + + Map resultMap = jedis.hgetAll(name); + + resultMap.get("0"); + + + String[] split = name.split("_"); + String i = null; + String j =null; + try { + i = split[1]; + j = split[2]; + } + catch (Exception e){ + e.printStackTrace(); + } + String firstValue = resultMap.get("0"); + String secondValue = resultMap.get("1"); + + + + + + +// firstValue = inputPath + firstValue; +// secondValue = inputPath + secondValue; + +// String[] firstValueSplit = firstValue.split("/"); +// String[] secondValueSplit = secondValue.split("/"); +// +// if (firstValueSplit.length == 1) { +// firstValue = inputPath + firstValueSplit[0]; +// } else { +// firstValue = inputPath + firstValueSplit[1]; +// } +// +// if (secondValueSplit.length == 1) { +// secondValue = inputPath + secondValueSplit[0]; +// } else { +// secondValue = inputPath + secondValueSplit[1]; +// } + + try { + Pair oldPair = getTree(firstValue); + Pair newPair = getTree(secondValue); + + ITree oldTree = oldPair.getValue0(); + ITree newTree = newPair.getValue0(); + + String oldProject = oldPair.getValue1(); + String newProject = newPair.getValue1(); + + + + Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); + m.match(); + + + ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); + ag.generate(); + List actions = ag.getActions(); + + String resultKey = "result_" + (String.valueOf(i)) + "_" + String.valueOf(j); + double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree); + String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1); + double diceSimilarity1 = m.diceSimilarity(oldTree, newTree); + String diceSimilarity = String.format("%1.2f", diceSimilarity1); + double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree); + String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); + + String editDistance = String.valueOf(actions.size()); +// jedis.select(1); + String result = resultMap.get("0") + "," + oldProject +"," + resultMap.get("1") + "," +newProject+ "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; +// jedis.set(resultKey, result); + + if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) + || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { + String matchKey = "match-"+clusterName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); + jedis.select(1); + jedis.set(matchKey, result); + } + jedis.select(0); + String pairKey = "pair_" + (String.valueOf(i)) + "_" + String.valueOf(j); + jedis.del(pairKey); + +// log.info("Completed " + resultKey); + + }catch (Exception e){ + log.error(e.toString() + " {}",(name)); + + + } + + + + + } + } + + + + protected static List parseChangedSourceCodeWithGumTree2(File prevFile, File revFile) { + List actionSets = new ArrayList<>(); + // GumTree results + List gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile); + if (gumTreeResults == null) { + resultType = 1; + return null; + } else if (gumTreeResults.size() == 0){ + resultType = 2; + return actionSets; + } else { + // Regroup GumTre results. + List allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults); +// for (HierarchicalActionSet actionSet : allActionSets) { +// String astNodeType = actionSet.getAstNodeType(); +// if (astNodeType.endsWith("Statement") || "FieldDeclaration".equals(astNodeType)) { +// actionSets.add(actionSet); +// } +// } + + // Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part. + // variable effects range, sub-actions are these kinds of modification? +// actionSets.addAll(new ActionFilter().filterOutUselessActions(allActionSets)); + + ListSorter sorter = new ListSorter<>(allActionSets); + actionSets = sorter.sortAscending(); + + if (actionSets.size() == 0) { + resultType = 3; + } + + return actionSets; + } + } + + +/* +orginal calculate pairs, from all dumps of the projects + */ + public static void calculatePairs(String inputPath, String outputPath) { + File folder = new File(inputPath); + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List pjs = stream + .filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); + + List fileToCompare = new ArrayList<>(); + for (File pj : pjs) { + File[] files = pj.listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + return name.startsWith("ASTDumps"); + } + }); + Collections.addAll(fileToCompare, files[0].listFiles()); + + } + System.out.println("a"); +// compareAll(fileToCompare); + readMessageFiles(fileToCompare, outputPath); + } + + /* + pairs of each cluster + */ + public static void calculatePairsOfClusters(String inputPath, String outputPath) { + File folder = new File(inputPath); + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List pjs = stream + .filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); + + FileHelper.createDirectory(outputPath + "pairs/"); + for (File pj : pjs) { + File[] files = pj.listFiles(); + List fileList = Arrays.asList(files); + + readMessageFilesCluster(fileList, outputPath,inputPath,pj.getName()); + + } + + } + + public static void processMessages(String inputPath, String outputPath) { + File folder = new File(outputPath + "pairs_splitted/"); + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List pjs = stream + .filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); + FileHelper.createDirectory(outputPath + "comparison_splitted/"); + pjs.parallelStream() + .forEach(m -> coreLoop(m, outputPath,inputPath)); + } + + private static void readMessageFilesCluster(List folders, String outputPath,String inputPath,String cluster) { + + List treesFileNames = new ArrayList<>(); + + + for (File target : folders) { + + treesFileNames.add(target.toString()); + } + + log.info("Calculating pairs"); +// treesFileNames = treesFileNames.subList(0,100); + + String filename = "cluster" + cluster; + byte [] buf = new byte[0]; + String line = null; + try { + + FileOutputStream fos = new FileOutputStream(outputPath + "pairs/" +filename+".txt"); + DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); + + + + for (int i = 0; i < treesFileNames.size(); i++) { + for (int j = i + 1; j < treesFileNames.size(); j++) { + + + + line = String.valueOf(i) +"\t" + String.valueOf(j) + "\t" + treesFileNames.get(i).replace(inputPath,"") + "\t" + treesFileNames.get(j).replace(inputPath,"")+"\n"; + outStream.write(line.getBytes()); + + } + } + outStream.close(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + }catch (java.nio.BufferOverflowException e) { + log.error(line); + log.error(String.valueOf(buf.length)); + e.printStackTrace(); + } + + log.info("Done pairs"); + } + + + + public static ITree getSimpliedTree(String fn) { + ITree tree = null; + try { + FileInputStream fi = new FileInputStream(new File(fn)); + ObjectInputStream oi = new ObjectInputStream(fi); + tree = (ITree) oi.readObject(); + oi.close(); + fi.close(); + + + } catch (FileNotFoundException e) { + log.error("File not found"); + e.printStackTrace(); + } catch (IOException e) { + log.error("Error initializing stream"); + e.printStackTrace(); + } catch (ClassNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + +// tree.setLabel(""); + tree.setParent(null); +// List descendants = tree.getDescendants(); +// for (ITree descendant : descendants) { +// descendant.setLabel(""); +// } + + return tree; + + } + + + private static void coreLoop(File mes, String outputPath,String inputPath) { + try { + + log.info("Starting in coreLoop"); + + BufferedReader br = null; + String sCurrentLine = null; + BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "comparison_splitted/" + "output_" + mes.getName())); + + br = new BufferedReader( + new FileReader(mes)); + while ((sCurrentLine = br.readLine()) != null) { + String currentLine = sCurrentLine; + String[] split = currentLine.split("\t"); + String i = split[0]; + String j = split[1]; + String firstValue = split[2]; + String secondValue = split[3]; + + firstValue = inputPath + firstValue.split("GumTreeOutput2")[1]; + secondValue = inputPath + secondValue.split("GumTreeOutput2")[1]; + + ITree oldTree = getSimpliedTree(firstValue); + + ITree newTree = getSimpliedTree(secondValue); + + Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); + m.match(); + + ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); + ag.generate(); + List actions = ag.getActions(); + writer.write(String.valueOf(i)); + writer.write("\t"); + writer.write(String.valueOf(j)); + writer.write("\t"); + + writer.write(String.format("%1.2f", m.chawatheSimilarity(oldTree, newTree))); + writer.write("\t"); + writer.write(String.format("%1.2f", m.diceSimilarity(oldTree, newTree))); + writer.write("\t"); + writer.write(String.format("%1.2f", m.jaccardSimilarity(oldTree, newTree))); + writer.write("\t"); + writer.write(String.valueOf(actions.size())); + writer.write("\t"); + writer.write(firstValue); + writer.write("\t"); + writer.write(secondValue); + writer.write("\n"); + + + } + writer.close(); + } catch (FileNotFoundException e) { + log.error("File not found"); + e.printStackTrace(); + } catch (IOException e) { + log.error("Error initializing stream"); + e.printStackTrace(); + + } + log.info("Completed output_" + mes.getName()); + } + + private static void readMessageFiles(List folders, String outputPath) { + + List treesFileNames = new ArrayList<>(); + + + for (File target : folders) { + + treesFileNames.add(target.toString()); + } + FileHelper.createDirectory(outputPath + "pairs/"); + log.info("Calculating pairs"); +// treesFileNames = treesFileNames.subList(0,100); + byte [] buf = new byte[0]; + String line = null; + try { + + FileChannel rwChannel = new RandomAccessFile(outputPath + "pairs/" +"textfile.txt", "rw").getChannel(); + ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE); + int fileCounter = 0; + + + for (int i = 0; i < treesFileNames.size(); i++) { + for (int j = i + 1; j < treesFileNames.size(); j++) { + + + + line = String.valueOf(i) +"\t" + String.valueOf(j) + "\t" + treesFileNames.get(i).replace("/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2","") + "\t" + treesFileNames.get(j).replace("/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2","")+"\n"; + buf = line.getBytes(); + if(wrBuf.remaining() > 500) { + wrBuf.put(buf); + }else{ + log.info("Next pair dump"); + fileCounter++; + rwChannel = new RandomAccessFile(outputPath+"pairs/" +"textfile"+String.valueOf(fileCounter)+".txt", "rw").getChannel(); + wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE); + } + + + + + } + } + rwChannel.close(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + }catch (java.nio.BufferOverflowException e) { + log.error(line); + log.error(String.valueOf(buf.length)); + e.printStackTrace(); + } + + log.info("Done pairs"); + } + + static final JedisPoolConfig poolConfig = buildPoolConfig(); + + + private static JedisPoolConfig buildPoolConfig() { + final JedisPoolConfig poolConfig = new JedisPoolConfig(); + poolConfig.setMaxTotal(128); + poolConfig.setMaxIdle(128); + poolConfig.setMinIdle(16); + poolConfig.setTestOnBorrow(true); + poolConfig.setTestOnReturn(true); + poolConfig.setTestWhileIdle(true); + poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); + poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); + poolConfig.setNumTestsPerEvictionRun(3); + poolConfig.setBlockWhenExhausted(true); + + return poolConfig; + } + + + +// return msgFiles; +} + + + + diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/AkkaTreeLoader.java deleted file mode 100644 index 8c32167..0000000 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/AkkaTreeLoader.java +++ /dev/null @@ -1,655 +0,0 @@ -package edu.lu.uni.serval.FixPatternParser.violations; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.actor.Props; -import com.github.gumtreediff.actions.ActionGenerator; -import com.github.gumtreediff.actions.model.*; -import com.github.gumtreediff.matchers.Matcher; -import com.github.gumtreediff.matchers.Matchers; -import com.github.gumtreediff.tree.ITree; -import com.github.gumtreediff.tree.Tree; -import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap; -import edu.lu.uni.serval.MultipleThreadsParser.*; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; -import edu.lu.uni.serval.utils.FileHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; - -import java.io.*; -import java.nio.file.Files; -import java.time.Duration; -import java.util.*; -import java.util.concurrent.Executors; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Created by anilkoyuncu on 19/03/2018. - */ -public class AkkaTreeLoader { - - private static class StreamGobbler implements Runnable { - private InputStream inputStream; - private Consumer consumer; - - public StreamGobbler(InputStream inputStream, Consumer consumer) { - this.inputStream = inputStream; - this.consumer = consumer; - } - - @Override - public void run() { - new BufferedReader(new InputStreamReader(inputStream)).lines() - .forEach(consumer); - } - } - - private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class); - - - public static void loadRedis(String cmd,String serverWait){ - Process process; - - try { -// String comd = String.format(cmd, f.getAbsoluteFile()); - process = Runtime.getRuntime() - - .exec(cmd); - - - StreamGobbler streamGobbler = - new StreamGobbler(process.getInputStream(), System.out::println); - Executors.newSingleThreadExecutor().submit(streamGobbler); -// int exitCode = process.waitFor(); -// assert exitCode == 0; - Thread.sleep(Integer.valueOf(serverWait)); - - } catch (IOException e) { - e.printStackTrace(); - } - catch (InterruptedException e) { - e.printStackTrace(); - } - log.info("Load done"); - } - - public static void main(String[] args) { - - - String inputPath; -// String outputPath; - String port; - String portInner; -// String pairsCSVPath; - String importScript; -// String pairsCompletedPath; - String serverWait; -// String option; - String dbDir; - String chunkName; - String numOfWorkers; - if (args.length > 0) { - inputPath = args[0]; - portInner = args[1]; - serverWait = args[2]; -// option = args[4]; - chunkName = args[3]; - numOfWorkers = args[4]; - dbDir = args[5]; - port = args[6]; -// pairsCSVPath = args[3]; -// importScript = args[4]; -// pairsCompletedPath = args[3]; - } else { - inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; -// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/"; - port = "6399"; - portInner = "6380"; - serverWait = "10000"; -// option = "COMP"; -// pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test"; -// importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh"; -// pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed"; - chunkName ="chunk5.rdb"; - dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; - numOfWorkers = "1"; - - } - String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \nnumOfWorks %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,numOfWorkers,dbDir); - log.info(parameters); - -// if (option.equals("CALC")) { -// calculatePairs(inputPath, port); -// log.info("Calculate pairs done"); -// }else { - String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; - cmd = String.format(cmd, dbDir,"dumps.rdb",Integer.valueOf(port)); - loadRedis(cmd,serverWait); - - String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; - cmd = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner)); - loadRedis(cmd,serverWait); - - JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); - JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000); - - - - comparePairs(inputPath, innerPool,outerPool, serverWait,chunkName,dbDir,numOfWorkers); - - String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; - stopServer = String.format(stopServer,Integer.valueOf(portInner)); - loadRedis(stopServer,serverWait); -// } - - - } - - - public static void comparePairs(String inputPath, JedisPool innerPool,JedisPool outerPool,String serverWait,String chunkName, String dbDir,String numOfWorkers){ - - - ScanResult scan; - try (Jedis inner = innerPool.getResource()) { - while (inner.ping()== "PONG"){ - log.info("wait"); - } - - ScanParams sc = new ScanParams(); - //150000000 - sc.count(150000000); - sc.match("pair_*"); - - scan = inner.scan("0", sc); - int size = scan.getResult().size(); - log.info("Scanning " + String.valueOf(size)); - } - List result = scan.getResult(); - - -// ActorSystem system = null; -// ActorRef parsingActor = null; -// final WorkMessage msg = new WorkMessage(0, result,innerPort,inputPath,dbDir,serverWait); -// try { -// -// log.info("Akka begins..."); -// system = ActorSystem.create("Tree-System"); -// parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers),dbDir,innerPort,serverWait), "tree-actor"); -// parsingActor.tell(msg, ActorRef.noSender()); -// } catch (Exception e) { -// system.shutdown(); -// e.printStackTrace(); -// } -// greeter.tell(); - result - .parallelStream() - .forEach(m -> - { - Compare compare = new Compare(); - compare.coreCompare(m, innerPool, outerPool); - ; - } - ); - - - - -// } - - - - - } - -// private static void coreCompare(String name , String inputPath, String innerPort) { -// JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000); -// Map resultMap; -// try (Jedis jedis = pool.getResource()) { -// resultMap = jedis.hgetAll(name); -// } -// String[] split = name.split("_"); -// -// -// String i = split[1]; -// String j = split[2]; -// String firstValue = resultMap.get("0"); -// String secondValue = resultMap.get("1"); -// -// String[] firstValueSplit = firstValue.split("GumTreeOutput2"); -// String[] secondValueSplit = secondValue.split("GumTreeOutput2"); -// -// if (firstValueSplit.length == 1) { -// firstValue = inputPath + firstValueSplit[0]; -// } else { -// firstValue = inputPath + firstValueSplit[1]; -// } -// -// if (secondValueSplit.length == 1) { -// secondValue = inputPath + secondValueSplit[0]; -// } else { -// secondValue = inputPath + secondValueSplit[1]; -// } -// -// try { -// ITree oldTree = getSimpliedTree(firstValue); -// -// ITree newTree = getSimpliedTree(secondValue); -// -// Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); -// m.match(); -// -// -// ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); -// ag.generate(); -// List actions = ag.getActions(); -// -// double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree); -// String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1); -// double diceSimilarity1 = m.diceSimilarity(oldTree, newTree); -// String diceSimilarity = String.format("%1.2f", diceSimilarity1); -// double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree); -// String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); -// -// String editDistance = String.valueOf(actions.size()); -// -// String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; -// -// -// if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) -// || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { -// String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); -// -// try (Jedis jedis = pool.getResource()) { -// jedis.select(1); -// jedis.set(matchKey, result); -// } -// } -// -// -// try (Jedis jedis = pool.getResource()) { -// jedis.del("pair_"+ (String.valueOf(i)) + "_" + String.valueOf(j)); -// } -// -// -// -// -// }catch (Exception e){ -// log.error(e.toString() + " {}",(name)); -// -// -// } -// -// -// -// -// -// } - - - - public static void calculatePairs(String inputPath,String port) { - File folder = new File(inputPath); - File[] listOfFiles = folder.listFiles(); - Stream stream = Arrays.stream(listOfFiles); - List pjs = stream - .filter(x -> !x.getName().startsWith(".")) - .collect(Collectors.toList()); - - List fileToCompare = new ArrayList<>(); - for (File pj : pjs) { - File[] files = pj.listFiles(new FilenameFilter() { - public boolean accept(File dir, String name) { - return name.startsWith("ActionSetDumps"); - } - }); - Collections.addAll(fileToCompare, files[0].listFiles()); - - } - System.out.println("a"); -// compareAll(fileToCompare); - readMessageFiles(fileToCompare,port); - } - -// public static void processMessages(String inputPath, String outputPath) { -// File folder = new File(outputPath + "pairs_splitted/"); -// File[] listOfFiles = folder.listFiles(); -// Stream stream = Arrays.stream(listOfFiles); -// List pjs = stream -// .filter(x -> !x.getName().startsWith(".")) -// .collect(Collectors.toList()); -// FileHelper.createDirectory(outputPath + "comparison_splitted/"); -// pjs.parallelStream() -// .forEach(m -> coreLoop(m, outputPath,inputPath)); -// } - - /** Read the object from Base64 string. */ - private static Object fromString( String s ) throws IOException , - ClassNotFoundException { - byte [] data = Base64.getDecoder().decode( s ); - ObjectInputStream ois = new ObjectInputStream( - new ByteArrayInputStream( data ) ); - Object o = ois.readObject(); - ois.close(); - return o; - } - public static ITree getSimpliedTree(String fn,JedisPool outerPool) { - - ITree tree = null; - Jedis inner = null; - try { - inner = outerPool.getResource(); - String s = inner.get(fn.substring(1)); - HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s); - - ITree parent = null; - ITree children =null; - tree = getASTTree(actionSet, parent, children); - tree.setParent(null); - - } catch (IOException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - }finally { - if (inner != null) { - inner.close(); - } - } - return tree; - - } - - public static List getKeysByValue(Map map, E value) { - return map.entrySet() - .stream() - .filter(entry -> Objects.equals(entry.getValue(), value)) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - } - - public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children){ - - int newType = 0; - - String astNodeType = actionSet.getAstNodeType(); - List keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType); - - if(keysByValue.size() != 1){ - log.error("Birden cok astnodemapmapping"); - } - newType = keysByValue.get(0); - if(actionSet.getParent() == null){ - //root - - parent = new Tree(newType,""); - }else{ - children = new Tree(newType,""); - parent.addChild(children); - } - List subActions = actionSet.getSubActions(); - if (subActions.size() != 0){ - for (HierarchicalActionSet subAction : subActions) { - - if(actionSet.getParent() == null){ - children = parent; - } - getASTTree(subAction,children,null); - - } - - - } - return parent; - } - - public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children){ - - int newType = 0; - - Action action = actionSet.getAction(); - if (action instanceof Update){ - newType = 101; - }else if(action instanceof Insert){ - newType =100; - }else if(action instanceof Move){ - newType = 102; - }else if(action instanceof Delete){ - newType=103; - }else{ - new Exception("unknow action"); - } - if(actionSet.getParent() == null){ - //root - - parent = new Tree(newType,""); - }else{ - children = new Tree(newType,""); - parent.addChild(children); - } - List subActions = actionSet.getSubActions(); - if (subActions.size() != 0){ - for (HierarchicalActionSet subAction : subActions) { - - if(actionSet.getParent() == null){ - children = parent; - } - getActionTree(subAction,children,null); - - } - - - } - return parent; - } - - - -// private static void coreLoop(File mes, String outputPath,String inputPath) { -// try { -// -// log.info("Starting in coreLoop"); -// -// BufferedReader br = null; -// String sCurrentLine = null; -// BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "comparison_splitted/" + "output_" + mes.getName())); -// -// br = new BufferedReader( -// new FileReader(mes)); -// while ((sCurrentLine = br.readLine()) != null) { -// String currentLine = sCurrentLine; -// String[] split = currentLine.split("\t"); -// String i = split[0]; -// String j = split[1]; -// String firstValue = split[2]; -// String secondValue = split[3]; -// -// firstValue = inputPath + firstValue.split("GumTreeOutput2")[1]; -// secondValue = inputPath + secondValue.split("GumTreeOutput2")[1]; -// -// ITree oldTree = getSimpliedTree(firstValue); -// -// ITree newTree = getSimpliedTree(secondValue); -// -// Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); -// m.match(); -// -// ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); -// ag.generate(); -// List actions = ag.getActions(); -// writer.write(String.valueOf(i)); -// writer.write("\t"); -// writer.write(String.valueOf(j)); -// writer.write("\t"); -// -// writer.write(String.format("%1.2f", m.chawatheSimilarity(oldTree, newTree))); -// writer.write("\t"); -// writer.write(String.format("%1.2f", m.diceSimilarity(oldTree, newTree))); -// writer.write("\t"); -// writer.write(String.format("%1.2f", m.jaccardSimilarity(oldTree, newTree))); -// writer.write("\t"); -// writer.write(String.valueOf(actions.size())); -// writer.write("\t"); -// writer.write(firstValue); -// writer.write("\t"); -// writer.write(secondValue); -// writer.write("\n"); -// -// -// } -// writer.close(); -// } catch (FileNotFoundException e) { -// log.error("File not found"); -// e.printStackTrace(); -// } catch (IOException e) { -// log.error("Error initializing stream"); -// e.printStackTrace(); -// -// } -// log.info("Completed output_" + mes.getName()); -// } - - private static void readMessageFiles(List folders,String port) { - - List treesFileNames = new ArrayList<>(); - - - for (File target : folders) { - - treesFileNames.add(target.toString()); - } -// FileHelper.createDirectory(outputPath + "pairs/"); - log.info("Calculating pairs"); -// treesFileNames = treesFileNames.subList(0,100); - byte [] buf = new byte[0]; - String line = null; - - -// FileChannel rwChannel = new RandomAccessFile(outputPath + "pairs/" +"textfile.txt", "rw").getChannel(); -// ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE); - int fileCounter = 0; - - JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); - try (Jedis jedis = jedisPool.getResource()) { - List dir = null; - List path = null; - for (int i = 0; i < treesFileNames.size(); i++) { - for (int j = i + 1; j < treesFileNames.size(); j++) { - - - // do operations with jedis resource - - String key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j); -// String value = treesFileNames.get(i).split("GumTreeOutput2")[1] +","+treesFileNames.get(j).split("GumTreeOutput2")[1]; -// jedis.set(key,value); - - jedis.hset(key,"0",treesFileNames.get(i).split("GumTreeOutput2")[1]); - jedis.hset(key,"1",treesFileNames.get(j).split("GumTreeOutput2")[1]); - //10000000 - if(Integer.compare(jedis.dbSize().intValue(),10000000) == 0){ - dir = jedis.configGet("dir"); - path = jedis.configGet("dbfilename"); - File dbPath = new File(dir.get(1)+"/"+path.get(1)); - File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb"); - try { - jedis.save(); - while (jedis.ping()== "PONG"){ - log.info("wait"); - } - - - - Files.copy(dbPath.toPath(),savePath.toPath()); - } catch (IOException e) { - - e.printStackTrace(); - } - fileCounter++; - jedis.flushDB(); - - } - - - } - } - jedis.save(); - fileCounter++; - File dbPath = new File(dir.get(1)+"/"+path.get(1)); - File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb"); - try { - - while (jedis.ping()== "PONG"){ - log.info("wait"); - } - - - - Files.copy(dbPath.toPath(),savePath.toPath()); - } catch (IOException e) { - - e.printStackTrace(); - } - jedis.flushDB(); - } - - - - - log.info("Done pairs"); - } - - static final JedisPoolConfig poolConfig = buildPoolConfig(); - - - private static JedisPoolConfig buildPoolConfig() { - final JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(128); - poolConfig.setMaxIdle(128); - poolConfig.setMinIdle(16); - poolConfig.setTestOnBorrow(false); - poolConfig.setTestOnReturn(false); - poolConfig.setTestWhileIdle(true); - poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); - poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); -// poolConfig.setNumTestsPerEvictionRun(3); - poolConfig.setBlockWhenExhausted(true); - - return poolConfig; - } - - private static List getMessageFiles(String gumTreeInput) { - String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile - File revFilesPath = new File(inputPath + "revFiles/"); - File[] revFiles = revFilesPath.listFiles(); // project folders - List msgFiles = new ArrayList<>(); - if (revFiles.length >= 0) { - for (File revFile : revFiles) { -// if (revFile.getName().endsWith(".java")) { - String fileName = revFile.getName(); - File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file - fileName = fileName.replace(".java", ".txt"); - File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file - File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file - edu.lu.uni.serval.MultipleThreadsParser.MessageFile msgFile = new edu.lu.uni.serval.MultipleThreadsParser.MessageFile(revFile, prevFile, diffentryFile); - msgFile.setPositionFile(positionFile); - msgFiles.add(msgFile); -// } - } - - return msgFiles; - } - else{ - return null; - } - } - - - -// return msgFiles; -} - - - - diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/RunnableCompare.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/RunnableCompare.java index 3079eb3..7f75eb5 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/RunnableCompare.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/RunnableCompare.java @@ -1,10 +1,8 @@ package edu.lu.uni.serval.FixPatternParser.violations; -import edu.lu.uni.serval.FixPatternParser.Parser; +import edu.lu.uni.serval.FixPatternParser.cluster.Compare; import redis.clients.jedis.JedisPool; -import java.io.File; - public class RunnableCompare implements Runnable { private String name; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeActor.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeActor.java index c9f0edb..4b74489 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeActor.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeActor.java @@ -6,13 +6,12 @@ import akka.actor.UntypedActor; import akka.japi.Creator; import akka.routing.RoundRobinPool; -import edu.lu.uni.serval.FixPatternParser.violations.WorkMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; -import static edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader.loadRedis; +import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; public class TreeActor extends UntypedActor { diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeWorker.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeWorker.java index f6839b5..37d5226 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeWorker.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeWorker.java @@ -3,35 +3,14 @@ package edu.lu.uni.serval.FixPatternParser.violations; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; -import com.github.gumtreediff.actions.ActionGenerator; -import com.github.gumtreediff.actions.model.Action; -import com.github.gumtreediff.matchers.Matcher; -import com.github.gumtreediff.matchers.Matchers; -import com.github.gumtreediff.tree.ITree; -import edu.lu.uni.serval.FixPatternParser.RunnableParser; -import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; -import edu.lu.uni.serval.FixPatternParser.violations.WorkMessage; -import edu.lu.uni.serval.config.Configuration; -import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.FixPatternParser.cluster.Compare; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.StringReader; import java.time.Duration; import java.util.List; -import java.util.Map; -import java.util.concurrent.*; - -import static edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader.comparePairs; -import static edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader.loadRedis; -import static edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader.getSimpliedTree; public class TreeWorker extends UntypedActor { private static Logger log = LoggerFactory.getLogger(TreeWorker.class);