diff --git a/pom.xml b/pom.xml index 36579b2..016c216 100644 --- a/pom.xml +++ b/pom.xml @@ -140,8 +140,8 @@ - edu.lu.uni.serval.Launcher - + edu.lu.uni.serval.fixminer.Launcher + diff --git a/src/main/java/edu/lu/uni/serval/Launcher.java b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java similarity index 76% rename from src/main/java/edu/lu/uni/serval/Launcher.java rename to src/main/java/edu/lu/uni/serval/fixminer/Launcher.java index f1f170d..400f9a2 100644 --- a/src/main/java/edu/lu/uni/serval/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java @@ -1,10 +1,7 @@ -package edu.lu.uni.serval; +package edu.lu.uni.serval.fixminer; -import edu.lu.uni.serval.FixPatternParser.cluster.*; -import edu.lu.uni.serval.FixPatternParser.violations.CallShell; -import edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster; -import edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster3; import edu.lu.uni.serval.FixPatternParser.violations.TestHunkParser; +import edu.lu.uni.serval.fixminer.cluster.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,37 +21,40 @@ public class Launcher { Properties appProps = new Properties(); - String appConfigPath = "/Users/kui.liu/Downloads/app.properties";//args[0]; +// String appConfigPath = "/Users/kui.liu/Downloads/app.properties";//args[0]; // String appConfigPath = "/Users/anilkoyuncu/bugStudy/release/code/app.properties"; + String appConfigPath = args[0]; appProps.load(new FileInputStream(appConfigPath)); String portInner = appProps.getProperty("portInner","6380"); String serverWait = appProps.getProperty("serverWait", "50000"); String numOfWorkers = appProps.getProperty("numOfWorkers", "10"); String jobType = appProps.getProperty("jobType","ALL"); - String port = appProps.getProperty("port","6399"); + String portDumps = appProps.getProperty("portDumps","6399"); String pythonPath = appProps.getProperty("pythonPath","/Users/anilkoyuncu/bugStudy/code/python"); String datasetPath = appProps.getProperty("datasetPath","/Users/anilkoyuncu/bugStudy/dataset"); String pjName = appProps.getProperty("pjName","allDataset"); String dbNo = appProps.getProperty("dbNo","0"); String actionType = appProps.getProperty("actionType","ALL"); String threshold = appProps.getProperty("threshold","1"); + String cursor = appProps.getProperty("cursor","10000000"); + String chunk = appProps.getProperty("chunk","1.txt"); String parameters = String.format("\nportInner %s " + "\nserverWait %s \nnumOfWorkers %s " + "\njobType %s \nport %s " + "\npythonPath %s \ndatasetPath %s" + - "\npjName %s \ndbNo %s \nactionType %s \nthreshold %s" - , portInner, serverWait, numOfWorkers, jobType, port, pythonPath,datasetPath,pjName,dbNo,actionType,threshold); + "\npjName %s \ndbNo %s \nactionType %s \nthreshold %s \ncursor %s" + , portInner, serverWait, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,dbNo,actionType,threshold,cursor); log.info(parameters); - mainLaunch(portInner, serverWait, numOfWorkers, jobType, port, pythonPath,datasetPath,pjName,dbNo,actionType,threshold); + mainLaunch(portInner, serverWait, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,dbNo,actionType,threshold,cursor,chunk); } - public static void mainLaunch(String portInner,String serverWait, String numOfWorkers,String jobType,String port, String pythonPath, String datasetPath, String pjName, String dbNo, String actionType,String threshold){ + public static void mainLaunch(String portInner, String serverWait, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String dbNo, String actionType, String threshold, String cursor, String chunk){ String dbDir; @@ -75,38 +75,40 @@ public class Launcher { TestHunkParser.main(gumInput, gumOutput, numOfWorkers, pjName); break; case "CACHE": - StoreFile.main(gumOutput, portInner, serverWait, dbDir, actionType+dumpsName,actionType); + StoreFile.main(gumOutput, portDumps, serverWait, dbDir, actionType+dumpsName,actionType); break; case "COMP": - CalculatePairs.main(serverWait, dbDir, actionType+dumpsName, portInner, pairsPath+actionType, pjName+actionType); + CalculatePairs.main(serverWait, dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType); ImportPairs2DB.main(pairsPath+actionType, portInner, serverWait, dbDir,datasetPath); - - AkkaTreeLoader.main(portInner, serverWait, dbDir, pjName +actionType+".csv.rdb" , port, actionType+dumpsName); + break; + case "AKKA": + AkkaTreeLoader.main(portInner, serverWait, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,cursor); + break; case "LEVEL1": - level1(portInner, serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput); + level1(portInner, serverWait, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput); break; //CALC python abstractPatch.py to from cluster folder case "LEVEL2": - level2(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); + level2(serverWait, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); break; //CALC via python case "LEVEL3": - level3(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); + level3(serverWait, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); break; case "ALL": // TestHunkParser.main(gumInput, gumOutput, numOfWorkers, pjName); - StoreFile.main(gumOutput, portInner, serverWait, dbDir, actionType+dumpsName,actionType); - level1(portInner, serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput); - level2(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); - level3(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); +// StoreFile.main(gumOutput, portInner, serverWait, dbDir, actionType+dumpsName,actionType); +// level1(portInner, serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput); +// level2(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); +// level3(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); break; case "EXTRACTPATTERN": - PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,port,pythonPath,datasetPath,pjName,dbNo,actionType,threshold); + PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,portDumps,pythonPath,datasetPath,pjName,dbNo,actionType,threshold); break; case "GETPATTERN": - PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,port,pythonPath,datasetPath,pjName,dbNo,actionType,threshold); + PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,portDumps,pythonPath,datasetPath,pjName,dbNo,actionType,threshold); break; } } catch (Exception e) { diff --git a/src/main/java/edu/lu/uni/serval/PatternExtractor.java b/src/main/java/edu/lu/uni/serval/fixminer/PatternExtractor.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/PatternExtractor.java rename to src/main/java/edu/lu/uni/serval/fixminer/PatternExtractor.java index 99e8650..edd65c1 100644 --- a/src/main/java/edu/lu/uni/serval/PatternExtractor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/PatternExtractor.java @@ -1,19 +1,8 @@ -package edu.lu.uni.serval; +package edu.lu.uni.serval.fixminer; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; - - -import java.io.*; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - import edu.lu.uni.serval.FixPattern.utils.Checker; -import edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +10,18 @@ import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; -import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.getASTTree; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.getASTTree; /** * Created by anilkoyuncu on 02/08/2018. diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/AkkaTreeLoader.java similarity index 67% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/AkkaTreeLoader.java index 4119e3e..0c81786 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/AkkaTreeLoader.java @@ -1,11 +1,10 @@ -package edu.lu.uni.serval.FixPatternParser.cluster; +package edu.lu.uni.serval.fixminer.cluster; import com.github.gumtreediff.tree.ITree; -import com.github.gumtreediff.tree.Tree; import com.github.gumtreediff.tree.TreeContext; import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap; -import edu.lu.uni.serval.FixPatternParser.violations.CallShell; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.*; @@ -16,7 +15,8 @@ import java.util.*; import java.util.concurrent.Executors; import java.util.function.Consumer; import java.util.stream.Collectors; -import org.junit.Assert; + +import static edu.lu.uni.serval.fixminer.cluster.akka.AkkaTreeParser.akkaCompare; /** * Created by anilkoyuncu on 19/03/2018. @@ -71,7 +71,7 @@ public class AkkaTreeLoader { private static Consumer consumer = Assert::assertNotNull; - public static void main(String portInner,String serverWait,String dbDir,String chunkName,String port, String dumpsName) throws Exception { + public static void main(String portInner, String serverWait, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, String cursor) throws Exception { String parameters = String.format("\nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s \ndumpsName %s",portInner,serverWait,chunkName,dbDir,dumpsName); @@ -81,57 +81,115 @@ public class AkkaTreeLoader { CallShell cs = new CallShell(); String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmd1 = String.format(cmd, dbDir,dumpsName,Integer.valueOf(port)); - +// cs.runShell(cmd1,serverWait); String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmd2 = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner)); - + log.info(cmd1); + log.info(cmd2); +// cs.runShell(cmd2,serverWait); JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000); +// Jedis jedis = new Jedis(new URI("redis://localhost:"+port)); +// while (!jedis.ping().equals("PONG")){ +// log.info("wait"); +// } +// +// jedis = new Jedis(new URI("redis://localhost:"+portInner)); +// while (!jedis.ping().equals("PONG")){ +// log.info("wait"); +// } - comparePairs(innerPool,outerPool); - String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; - String stopServer1 = String.format(stopServer,Integer.valueOf(portInner)); +// String pairsIndexFile = pairsPath + "/"+ chunkName; +// pairsIndexFile = pairsIndexFile.replace("csv.rdb","index"); +// +// +// Pattern pattern = Pattern.compile(","); +// String csvFile = pairsIndexFile; +// try { +// try (BufferedReader in = new BufferedReader(new FileReader(csvFile));){ +// Map namefreq = in +// .lines() +// // .skip(1) +// .map(x -> pattern.split(x)) +// // .filter(x -> x[4].equals("CA") && x[3].equals("F")) +// .collect(HashMap::new, (map, x) -> +// map.put(x[0], x[1]), +// Map::putAll); +// // namefreq.forEach((k, v) -> System.out.println(k + " => " + v)); +// +// Jedis inner = null; +// try { +// inner = outerPool.getResource(); +// +// for (Map.Entry entry : namefreq.entrySet()) { +// String key = entry.getKey(); +// String value = entry.getValue(); +// inner.select(1); +// inner.set(key,value); +// // ... +// } +// +// +// }finally { +// if (inner != null) { +// inner.close(); +// } +// } +// +// +// } +// } catch (IOException e) { +// e.printStackTrace(); +// } - cs.runShell(stopServer1,serverWait); - stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; - String stopServer2 = String.format(stopServer,Integer.valueOf(port)); - cs.runShell(stopServer2,serverWait); + +// comparePairs(innerPool,outerPool); + akkaCompare(innerPool,outerPool,numOfWorkers,cursor); + +// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; +// String stopServer1 = String.format(stopServer,Integer.valueOf(portInner)); +// +// cs.runShell(stopServer1,serverWait); +// stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; +// String stopServer2 = String.format(stopServer,Integer.valueOf(port)); +// +// cs.runShell(stopServer2,serverWait); } - public static void comparePairs(JedisPool innerPool,JedisPool outerPool){ ScanResult scan; + try (Jedis inner = innerPool.getResource()) { - while (inner.ping()== "PONG"){ + while (!inner.ping().equals("PONG")){ log.info("wait"); } ScanParams sc = new ScanParams(); //150000000 - sc.count(150000000); + log.info("Scanning "); + sc.count(250000000); + sc.match("pair_[0-9]*"); scan = inner.scan("0", sc); int size = scan.getResult().size(); - log.info("Scanning " + String.valueOf(size)); + log.info("Scanned " + String.valueOf(size)); } List result = scan.getResult(); - - - + log.info("Getting results"); result - .parallelStream() - .forEach(m -> + .parallelStream() + .forEach(m -> { Compare compare = new Compare(); compare.coreCompare(m, innerPool, outerPool); @@ -165,14 +223,20 @@ public class AkkaTreeLoader { Jedis inner = null; try { inner = outerPool.getResource(); - String s = inner.get(fn); + while (!inner.ping().equals("PONG")){ + log.info("wait"); + } + inner.select(1); + String dist2load = inner.get(fn); + inner.select(0); + String s = inner.get(dist2load); HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s); ITree parent = null; ITree children =null; TreeContext tc = new TreeContext(); tree = getASTTree(actionSet, parent, children,tc); -// tree.setParent(null); + tree.setParent(null); tc.validate(); } catch (IOException e) { diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java similarity index 82% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java index f906103..2e25b25 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java @@ -1,6 +1,5 @@ -package edu.lu.uni.serval.FixPatternParser.cluster; +package edu.lu.uni.serval.fixminer.cluster; -import edu.lu.uni.serval.FixPatternParser.violations.CallShell; import edu.lu.uni.serval.utils.FileHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,7 +13,7 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.List; -import static edu.lu.uni.serval.FixPatternParser.cluster.TreeLoaderClusterL1.poolConfig; +import static edu.lu.uni.serval.fixminer.cluster.TreeLoaderClusterL1.poolConfig; /** * Created by anilkoyuncu on 05/04/2018. @@ -63,32 +62,42 @@ public class CalculatePairs { try { // FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv"); // DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); +// + FileOutputStream fosIndex = new FileOutputStream(outputPath + "/" +pjName+".index"); + DataOutputStream outStreamIndex = new DataOutputStream(new BufferedOutputStream(fosIndex)); +// // // // // for (int i = 0; i < result.size(); i++) { +// line = String.valueOf(i) +"," + result.get(i)+"\n"; +// outStreamIndex.write(line.getBytes()); +// // for (int j = i + 1; j < result.size(); j++) { // // // -// line = String.valueOf(i) +"," + String.valueOf(j) + "," + result.get(i) + "," + result.get(j)+"\n"; +// line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n"; // outStream.write(line.getBytes()); // // } // } // outStream.close(); +// outStreamIndex.close(); int fileCounter = 0; FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +pjName +String.valueOf(fileCounter)+".txt", "rw").getChannel(); - int maxSize = 500*500000; + int maxSize = 500*1000000; ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, maxSize); for (int i = 0; i < result.size(); i++) { + line = String.valueOf(i) +"," + result.get(i)+"\n"; + outStreamIndex.write(line.getBytes()); for (int j = i + 1; j < result.size(); j++) { - line = String.valueOf(i) +"\t" + String.valueOf(j) + "\t" + result.get(i) + "\t" + result.get(j)+"\n"; + line = String.valueOf(i) +"\t" + String.valueOf(j)+"\n"; // + "\t" + result.get(i) + "\t" + result.get(j)+"\n"; buf = line.getBytes(); if(wrBuf.remaining() > 500) { wrBuf.put(buf); @@ -104,6 +113,7 @@ public class CalculatePairs { } } + outStreamIndex.close(); rwChannel.close(); } catch (FileNotFoundException e) { e.printStackTrace(); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/CallShell.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java similarity index 89% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/CallShell.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java index 07e0398..d108dcf 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/CallShell.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java @@ -1,9 +1,11 @@ -package edu.lu.uni.serval.FixPatternParser.violations; +package edu.lu.uni.serval.fixminer.cluster; /** * Created by anilkoyuncu on 17/04/2018. */ -import java.io.*; + +import java.io.BufferedReader; +import java.io.InputStreamReader; public class CallShell { diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/Compare.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java similarity index 78% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/Compare.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java index 1b3d5cb..64900a3 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/Compare.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser.cluster; +package edu.lu.uni.serval.fixminer.cluster; import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.model.Action; @@ -13,7 +13,7 @@ import redis.clients.jedis.JedisPool; import java.util.List; import java.util.Map; -import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.getSimpliedTree; +import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.getSimpliedTree; /** * Created by anilkoyuncu on 03/04/2018. @@ -23,7 +23,7 @@ public class Compare { private Logger log = LoggerFactory.getLogger(Compare.class); - public void coreCompare(String name , JedisPool innerPool, JedisPool outerPool) { + public void coreCompare(String name, JedisPool innerPool, JedisPool outerPool) { Map resultMap; Jedis jedis = null; @@ -32,19 +32,21 @@ public class Compare { try { jedis = innerPool.getResource(); - resultMap = jedis.hgetAll(name); +// resultMap = jedis.hgetAll(name); String[] split = name.split("_"); String i = split[1]; String j = split[2]; - String firstValue = resultMap.get("0"); - String secondValue = resultMap.get("1"); - oldTree = getSimpliedTree(firstValue,outerPool); - newTree = getSimpliedTree(secondValue,outerPool); +// String firstValue = resultMap.get("0"); +// String secondValue = resultMap.get("1"); + + oldTree = getSimpliedTree(i,outerPool); + + newTree = getSimpliedTree(j,outerPool); Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); m.match(); @@ -63,7 +65,7 @@ public class Compare { String editDistance = String.valueOf(actions.size()); - String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; + String result = i + "," + j + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) @@ -77,7 +79,7 @@ public class Compare { } - + jedis.select(0); jedis.del("pair_" + (String.valueOf(i)) + "_" + String.valueOf(j)); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/ImportPairs2DB.java similarity index 84% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/ImportPairs2DB.java index 3571263..fab81b6 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/ImportPairs2DB.java @@ -1,18 +1,16 @@ -package edu.lu.uni.serval.FixPatternParser.cluster; +package edu.lu.uni.serval.fixminer.cluster; -import edu.lu.uni.serval.FixPatternParser.violations.CallShell; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; -//import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; -//import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedisWait; +//import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis; +//import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedisWait; /** diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoader.java similarity index 99% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoader.java index 197a573..baa1c42 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoader.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser.violations; +package edu.lu.uni.serval.fixminer.cluster; import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.model.*; @@ -14,11 +14,7 @@ import org.slf4j.LoggerFactory; import redis.clients.jedis.*; import java.io.*; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.nio.file.Files; -import java.nio.file.StandardCopyOption; -import java.sql.Timestamp; import java.time.Duration; import java.util.*; import java.util.concurrent.Executors; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster.java index 4282c0c..909b1b3 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser.violations; +package edu.lu.uni.serval.fixminer.cluster; import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.model.*; @@ -33,14 +33,14 @@ public class MultiThreadTreeLoaderCluster { CallShell cs = new CallShell(); String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner)); -// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000"); +// edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000"); cs.runShell(cmd1,serverWait); String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port)); cs.runShell(cmd2,serverWait); -// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000"); +// edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000"); String cmd3; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster3.java similarity index 95% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster3.java index c35d5f5..149ed53 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster3.java @@ -1,41 +1,25 @@ -package edu.lu.uni.serval.FixPatternParser.violations; - -import static edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader.getKeysByValue; -import static edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster.fromString; - -import java.io.BufferedOutputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; +package edu.lu.uni.serval.fixminer.cluster; +import com.github.gumtreediff.tree.ITree; +import com.github.gumtreediff.tree.TreeContext; +import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.utils.FileHelper; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.similarity.JaroWinklerDistance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import redis.clients.jedis.*; -import com.github.gumtreediff.tree.ITree; -import com.github.gumtreediff.tree.TreeContext; +import java.io.*; +import java.time.Duration; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; -import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap; -import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; -import edu.lu.uni.serval.utils.FileHelper; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; -import redis.clients.jedis.ScanParams; -import redis.clients.jedis.ScanResult; +import static edu.lu.uni.serval.fixminer.cluster.MultiThreadTreeLoader.getKeysByValue; +import static edu.lu.uni.serval.fixminer.cluster.MultiThreadTreeLoaderCluster.fromString; /** * Created by anilkoyuncu on 19/03/2018. @@ -52,12 +36,12 @@ public class MultiThreadTreeLoaderCluster3 { CallShell cs = new CallShell(); String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner)); -// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000"); +// edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000"); cs.runShell(cmd1,serverWait); String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port)); -// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000"); +// edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000"); cs.runShell(cmd2,serverWait); String cmd3; @@ -95,7 +79,7 @@ public class MultiThreadTreeLoaderCluster3 { if (size == 0) { String comd = String.format(cmd3, f.getPath(), portInner); cs.runShell(comd); -// edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster. +// edu.lu.uni.serval.fixminer.cluster.MultiThreadTreeLoaderCluster. // loadRedis(comd); scan = jedis.scan("0", sc); @@ -569,20 +553,22 @@ public class MultiThreadTreeLoaderCluster3 { CharSequence[] oldSequences = oldTokens.toArray(new CharSequence[oldTokens.size()]); CharSequence[] newSequences = newTokens.toArray(new CharSequence[newTokens.size()]); JaroWinklerDistance jwd = new JaroWinklerDistance(); - LevenshteinDistance ld = new LevenshteinDistance(); +// LevenshteinDistance ld = new LevenshteinDistance(); - Double overallSimi = Double.valueOf(1); + Double overallSimi = Double.valueOf(0); if(oldSequences.length > 0 && (oldSequences.length == newSequences.length)){ for (int idx = 0; idx < newSequences.length; idx++) { Double simi = jwd.apply(newSequences[idx], oldSequences[idx]); - overallSimi = simi * overallSimi; + overallSimi = simi + overallSimi; } + overallSimi = overallSimi / oldSequences.length; }else{ overallSimi = Double.valueOf(0); // if(oldSequences.length != 0) { // log.info("ERROR"); // } } + int retval = Double.compare(overallSimi, Double.valueOf(0.8)); if(retval >= 0){ diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/RunnableCompare.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java similarity index 81% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/RunnableCompare.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java index 7f75eb5..f51ca09 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/RunnableCompare.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java @@ -1,6 +1,5 @@ -package edu.lu.uni.serval.FixPatternParser.violations; +package edu.lu.uni.serval.fixminer.cluster; -import edu.lu.uni.serval.FixPatternParser.cluster.Compare; import redis.clients.jedis.JedisPool; public class RunnableCompare implements Runnable { diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreFile.java similarity index 93% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreFile.java index d89e87b..977b701 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreFile.java @@ -1,14 +1,10 @@ -package edu.lu.uni.serval.FixPatternParser.cluster; +package edu.lu.uni.serval.fixminer.cluster; -import edu.lu.uni.serval.FixPatternParser.violations.CallShell; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; -import org.apache.commons.lang3.ArrayUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; -import redis.clients.jedis.ScanResult; import java.io.*; import java.util.ArrayList; @@ -18,8 +14,7 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; -import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; -import static edu.lu.uni.serval.FixPatternParser.cluster.TreeLoaderClusterL1.poolConfig; +import static edu.lu.uni.serval.fixminer.cluster.TreeLoaderClusterL1.poolConfig; /** * Created by anilkoyuncu on 03/04/2018. diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeActor.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeActor.java similarity index 56% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeActor.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeActor.java index 4b74489..443d889 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeActor.java @@ -1,17 +1,19 @@ -package edu.lu.uni.serval.FixPatternParser.violations; +package edu.lu.uni.serval.fixminer.cluster; import akka.actor.ActorRef; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; import akka.routing.RoundRobinPool; - +import edu.lu.uni.serval.fixminer.cluster.akka.TreeMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; import java.util.List; -import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; +import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis; public class TreeActor extends UntypedActor { @@ -21,20 +23,14 @@ public class TreeActor extends UntypedActor { private final int numberOfWorkers; private int counter = 0; - private String innerPort; - private String dbDir; - private String serverWait; - public TreeActor(int numberOfWorkers,String dbDir,String innerPort,String serverWait) { + public TreeActor(int numberOfWorkers) { mineRouter = this.getContext().actorOf(new RoundRobinPool(numberOfWorkers) .props(TreeWorker.props()), "tree-router"); this.numberOfWorkers = numberOfWorkers; - this.innerPort = innerPort; - this.dbDir = dbDir; - this.serverWait = serverWait; } - public static Props props(final int numberOfWorkers, final String dbDir,final String innerPort, final String serverWait) { + public static Props props(final int numberOfWorkers) { return Props.create(new Creator() { @@ -42,7 +38,7 @@ public class TreeActor extends UntypedActor { @Override public TreeActor create() throws Exception { - return new TreeActor(numberOfWorkers,dbDir,innerPort,serverWait); + return new TreeActor(numberOfWorkers); } }); @@ -51,14 +47,13 @@ public class TreeActor extends UntypedActor { @SuppressWarnings("deprecation") @Override public void onReceive(Object message) throws Exception { - if (message instanceof WorkMessage) { - List files = ((WorkMessage) message).getMsgFiles(); - String innerPort = ((WorkMessage) message).getInnerPort(); - String inputPath = ((WorkMessage) message).getInputPath(); - String dbDir = ((WorkMessage) message).getDbDir(); - String serverWait = ((WorkMessage) message).getServerWait(); + if (message instanceof TreeMessage) { + List pairs = ((TreeMessage) message).getName(); + JedisPool innerPool = ((TreeMessage) message).getInnerPool(); + JedisPool outerPool = ((TreeMessage) message).getOuterPool(); - int size = files.size(); + + int size = pairs.size(); int average = size / numberOfWorkers; int reminder = size % numberOfWorkers; int counter = 0; @@ -68,8 +63,8 @@ public class TreeActor extends UntypedActor { if (counter < reminder) counter ++; int toIndex = (i + 1) * average + counter; - List filesOfWorkers = files.subList(fromIndex, toIndex); - final WorkMessage workMsg = new WorkMessage(i + 1, filesOfWorkers,innerPort,inputPath,dbDir,serverWait); + List pairsOfWorkers = pairs.subList(fromIndex, toIndex); + final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool); mineRouter.tell(workMsg, getSelf()); logger.info("Assign a task to worker #" + (i + 1) + "..."); } @@ -81,9 +76,7 @@ public class TreeActor extends UntypedActor { this.getContext().stop(mineRouter); this.getContext().stop(getSelf()); this.getContext().system().shutdown(); - String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; - stopServer = String.format(stopServer,Integer.valueOf(innerPort)); - loadRedis(stopServer,serverWait); + } } else { unhandled(message); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeLoaderClusterL1.java similarity index 97% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeLoaderClusterL1.java index 07ca3af..6435e58 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeLoaderClusterL1.java @@ -1,6 +1,5 @@ -package edu.lu.uni.serval.FixPatternParser.cluster; +package edu.lu.uni.serval.fixminer.cluster; -import edu.lu.uni.serval.FixPatternParser.violations.CallShell; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.*; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeWorker.java similarity index 63% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeWorker.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeWorker.java index 37d5226..4c0e6a0 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeWorker.java @@ -1,24 +1,51 @@ -package edu.lu.uni.serval.FixPatternParser.violations; +package edu.lu.uni.serval.fixminer.cluster; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; - -import edu.lu.uni.serval.FixPatternParser.cluster.Compare; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.fixminer.cluster.akka.TreeMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; import java.time.Duration; import java.util.List; +import java.util.concurrent.*; + +import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.poolConfig; public class TreeWorker extends UntypedActor { private static Logger log = LoggerFactory.getLogger(TreeWorker.class); +// private JedisPool innerPool; +// private JedisPool outerPool; +// +// public TreeWorker(String innerPort,String outerPort) { +//// this.innerPool = innerPool; +//// this.outerPool = outerPool; +// this.outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(outerPort),20000000); +// this.innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(innerPort),20000000); +// +// +// } +// +// public static Props props(final String innerPort,final String outerPort) { +// return Props.create(new Creator() { +// +// private static final long serialVersionUID = -7615153844097275009L; +// +// @Override +// public TreeWorker create() throws Exception { +// return new TreeWorker(innerPort,outerPort); +// } +// +// }); +// } - public TreeWorker() { - + public TreeWorker() { } @@ -35,47 +62,48 @@ public class TreeWorker extends UntypedActor { }); } + + @Override public void onReceive(Object message) throws Exception { - if(message instanceof edu.lu.uni.serval.FixPatternParser.violations.WorkMessage) { + if(message instanceof TreeMessage) { // if (message instanceof edu.lu.uni.serval.MultipleThreadsParser.WorkMessage) { - edu.lu.uni.serval.FixPatternParser.violations.WorkMessage msg = (WorkMessage) message; - List files = msg.getMsgFiles(); - String innerPort = msg.getInnerPort(); - String inputPath = msg.getInputPath(); - String dbDir = msg.getDbDir(); - String serverWait = msg.getServerWait(); + TreeMessage msg = (TreeMessage) message; + List files = msg.getName(); + JedisPool innerPool = msg.getInnerPool(); + JedisPool outerPool = msg.getOuterPool(); + int id = msg.getId(); - int counter = new Object() { +// int counter = new Object() { int counter = 0; // -// for (String name : files) + for (String name : files) { - files.stream(). - parallel(). - peek(x -> counter++). - forEach(m -> - { - Compare compare = new Compare(); -// compare.coreCompare(m, inputPath, innerPort); - } - ); - } - }.counter; +// files.stream(). +// parallel(). +// peek(x -> counter++). +// forEach(m -> +// { +// Compare compare = new Compare(); +// compare.coreCompare(m, innerPool, outerPool); +// } +// ); +// } +// }.counter; // -// final ExecutorService executor = Executors.newFixedThreadPool(20); + final ExecutorService executor = Executors.newFixedThreadPool(20); // // schedule the work -// final Future future = executor.submit(new RunnableCompare(name, inputPath, innerPort, new Compare(poolConfig))); -// try { - // wait for task to complete -// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); -// Compare compare = new Compare(poolConfig); -// compare.coreCompare(name, inputPath, innerPort); -// counter++; + final Future future = executor.submit(new RunnableCompare(name, innerPool, outerPool, new Compare())); + try { +// wait for task to complete + future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); + Compare compare = new Compare(); + compare.coreCompare(name, innerPool, outerPool); + counter++; // nullDiffEntry += parser.nullMatchedDiffEntry; // nullMappingGumTreeResults += parser.nullMappingGumTreeResult; // pureDeletion += parser.pureDeletions; @@ -115,22 +143,22 @@ public class TreeWorker extends UntypedActor { // testingInfo.setLength(0); // } // } -// } catch (TimeoutException e) { -// future.cancel(true); + } catch (TimeoutException e) { + future.cancel(true); ////// timeouts += countAlarms(positionFile, "#Timeout:"); // System.err.println("#Timeout: " + name); -// } catch (InterruptedException e) { + } catch (InterruptedException e) { ////// timeouts += countAlarms(positionFile, "#TimeInterrupted:"); //// System.err.println("#TimeInterrupted: " + revFile.getName()); -// e.printStackTrace(); -// } catch (ExecutionException e) { + e.printStackTrace(); + } catch (ExecutionException e) { ////// timeouts += countAlarms(positionFile, "#TimeAborted:"); //// System.err.println("#TimeAborted: " + revFile.getName()); -// e.printStackTrace(); -// } finally { -// executor.shutdownNow(); -// } -// } + e.printStackTrace(); + } finally { + executor.shutdownNow(); + } + } log.info("bitti"); log.info("Worker #" + id +"finialized parsing " + counter + " files..."); @@ -169,24 +197,24 @@ public class TreeWorker extends UntypedActor { } } - static final JedisPoolConfig poolConfig = buildPoolConfig(); - - - private static JedisPoolConfig buildPoolConfig() { - final JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(128); - poolConfig.setMaxIdle(128); - poolConfig.setMinIdle(16); - poolConfig.setTestOnBorrow(true); - poolConfig.setTestOnReturn(true); - poolConfig.setTestWhileIdle(true); - poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); - poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); - poolConfig.setNumTestsPerEvictionRun(3); - poolConfig.setBlockWhenExhausted(true); - - return poolConfig; - } +// static final JedisPoolConfig poolConfig = buildPoolConfig(); +// +// +// private static JedisPoolConfig buildPoolConfig() { +// final JedisPoolConfig poolConfig = new JedisPoolConfig(); +// poolConfig.setMaxTotal(128); +// poolConfig.setMaxIdle(128); +// poolConfig.setMinIdle(16); +// poolConfig.setTestOnBorrow(true); +// poolConfig.setTestOnReturn(true); +// poolConfig.setTestWhileIdle(true); +// poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); +// poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); +// poolConfig.setNumTestsPerEvictionRun(3); +// poolConfig.setBlockWhenExhausted(true); +// +// return poolConfig; +// } diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/WorkMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/WorkMessage.java similarity index 87% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/violations/WorkMessage.java rename to src/main/java/edu/lu/uni/serval/fixminer/cluster/WorkMessage.java index 6343c51..f009b25 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/WorkMessage.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/WorkMessage.java @@ -1,6 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser.violations; - -import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; +package edu.lu.uni.serval.fixminer.cluster; import java.util.List; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java new file mode 100644 index 0000000..d76b532 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java @@ -0,0 +1,75 @@ +package edu.lu.uni.serval.fixminer.cluster.akka; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import edu.lu.uni.serval.fixminer.cluster.TreeActor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.ScanParams; +import redis.clients.jedis.ScanResult; + +import java.util.List; + +/** + * Created by anilkoyuncu on 12/09/2018. + */ +public class AkkaTreeParser { + + private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class); + + + public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, String cursor){ + + final List listOfPairs = getMessages(innerPool,cursor); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" + + + + + ActorSystem system = null; + ActorRef parsingActor = null; + final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool); + try { + log.info("Akka begins..."); + system = ActorSystem.create("Compare-EnhancedDiff-System"); + + parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor"); + parsingActor.tell(msg, ActorRef.noSender()); + } catch (Exception e) { + system.shutdown(); + e.printStackTrace(); + } + } + + public static List getMessages(JedisPool innerPool, String cursor){ + + + ScanResult scan; + + try (Jedis inner = innerPool.getResource()) { + while (!inner.ping().equals("PONG")){ + log.info("wait"); + } + + ScanParams sc = new ScanParams(); + //150000000 + log.info("Scanning "); + sc.count(Integer.valueOf(cursor)); + + sc.match("pair_[0-9]*"); + + scan = inner.scan("0", sc); + int size = scan.getResult().size(); + log.info("Scanned " + String.valueOf(size)); + } + List result = scan.getResult(); + log.info("Getting results"); + return result; + + + + + + } +} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java new file mode 100644 index 0000000..b23bfe8 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java @@ -0,0 +1,54 @@ +package edu.lu.uni.serval.fixminer.cluster.akka; + +import redis.clients.jedis.JedisPool; + +import java.util.List; + +/** + * Created by anilkoyuncu on 12/09/2018. + */ +public class TreeMessage { + private int id; + private List name; + private JedisPool innerPool; + private JedisPool outerPool; + + public TreeMessage(int id, List name, JedisPool innerPool, JedisPool outerPool) { + this.id = id; + this.name = name; + this.innerPool = innerPool; + this.outerPool = outerPool; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public List getName() { + return name; + } + + public void setName(List name) { + this.name = name; + } + + public JedisPool getInnerPool() { + return innerPool; + } + + public void setInnerPool(JedisPool innerPool) { + this.innerPool = innerPool; + } + + public JedisPool getOuterPool() { + return outerPool; + } + + public void setOuterPool(JedisPool outerPool) { + this.outerPool = outerPool; + } +}