azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

akka version of tree comp

Browse Source
This commit is contained in:
fixminer
2018-09-13 16:18:06 +02:00
parent fc5f5b4089
commit 0d391e1ab9
19 changed files with 432 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pom.xml
+2 -2
View File
@@ -140,8 +140,8 @@
<archive> <archive>
<manifest> <manifest>
<mainClass> <mainClass>
edu.lu.uni.serval.Launcher edu.lu.uni.serval.fixminer.Launcher
</mainClass> </mainClass>
</manifest> </manifest>
</archive> </archive>
<descriptorRefs> <descriptorRefs>
src/main/java/edu/lu/uni/serval/Launcher.java → src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+26 -24
View File
@@ -1,10 +1,7 @@
package edu.lu.uni.serval; package edu.lu.uni.serval.fixminer;
import edu.lu.uni.serval.FixPatternParser.cluster.*;
import edu.lu.uni.serval.FixPatternParser.violations.CallShell;
import edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster;
import edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster3;
import edu.lu.uni.serval.FixPatternParser.violations.TestHunkParser; import edu.lu.uni.serval.FixPatternParser.violations.TestHunkParser;
import edu.lu.uni.serval.fixminer.cluster.*;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -24,37 +21,40 @@ public class Launcher {
Properties appProps = new Properties(); Properties appProps = new Properties();
String appConfigPath = "/Users/kui.liu/Downloads/app.properties";//args[0]; // String appConfigPath = "/Users/kui.liu/Downloads/app.properties";//args[0];
// String appConfigPath = "/Users/anilkoyuncu/bugStudy/release/code/app.properties"; // String appConfigPath = "/Users/anilkoyuncu/bugStudy/release/code/app.properties";
String appConfigPath = args[0];
appProps.load(new FileInputStream(appConfigPath)); appProps.load(new FileInputStream(appConfigPath));
String portInner = appProps.getProperty("portInner","6380"); String portInner = appProps.getProperty("portInner","6380");
String serverWait = appProps.getProperty("serverWait", "50000"); String serverWait = appProps.getProperty("serverWait", "50000");
String numOfWorkers = appProps.getProperty("numOfWorkers", "10"); String numOfWorkers = appProps.getProperty("numOfWorkers", "10");
String jobType = appProps.getProperty("jobType","ALL"); String jobType = appProps.getProperty("jobType","ALL");
String port = appProps.getProperty("port","6399"); String portDumps = appProps.getProperty("portDumps","6399");
String pythonPath = appProps.getProperty("pythonPath","/Users/anilkoyuncu/bugStudy/code/python"); String pythonPath = appProps.getProperty("pythonPath","/Users/anilkoyuncu/bugStudy/code/python");
String datasetPath = appProps.getProperty("datasetPath","/Users/anilkoyuncu/bugStudy/dataset"); String datasetPath = appProps.getProperty("datasetPath","/Users/anilkoyuncu/bugStudy/dataset");
String pjName = appProps.getProperty("pjName","allDataset"); String pjName = appProps.getProperty("pjName","allDataset");
String dbNo = appProps.getProperty("dbNo","0"); String dbNo = appProps.getProperty("dbNo","0");
String actionType = appProps.getProperty("actionType","ALL"); String actionType = appProps.getProperty("actionType","ALL");
String threshold = appProps.getProperty("threshold","1"); String threshold = appProps.getProperty("threshold","1");
String cursor = appProps.getProperty("cursor","10000000");
String chunk = appProps.getProperty("chunk","1.txt");
String parameters = String.format("\nportInner %s " + String parameters = String.format("\nportInner %s " +
"\nserverWait %s \nnumOfWorkers %s " + "\nserverWait %s \nnumOfWorkers %s " +
"\njobType %s \nport %s " + "\njobType %s \nport %s " +
"\npythonPath %s \ndatasetPath %s" + "\npythonPath %s \ndatasetPath %s" +
"\npjName %s \ndbNo %s \nactionType %s \nthreshold %s" "\npjName %s \ndbNo %s \nactionType %s \nthreshold %s \ncursor %s"
, portInner, serverWait, numOfWorkers, jobType, port, pythonPath,datasetPath,pjName,dbNo,actionType,threshold); , portInner, serverWait, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,dbNo,actionType,threshold,cursor);
log.info(parameters); log.info(parameters);
mainLaunch(portInner, serverWait, numOfWorkers, jobType, port, pythonPath,datasetPath,pjName,dbNo,actionType,threshold); mainLaunch(portInner, serverWait, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,dbNo,actionType,threshold,cursor,chunk);
} }
public static void mainLaunch(String portInner,String serverWait, String numOfWorkers,String jobType,String port, String pythonPath, String datasetPath, String pjName, String dbNo, String actionType,String threshold){ public static void mainLaunch(String portInner, String serverWait, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String dbNo, String actionType, String threshold, String cursor, String chunk){
String dbDir; String dbDir;
@@ -75,38 +75,40 @@ public class Launcher {
TestHunkParser.main(gumInput, gumOutput, numOfWorkers, pjName); TestHunkParser.main(gumInput, gumOutput, numOfWorkers, pjName);
break; break;
case "CACHE": case "CACHE":
StoreFile.main(gumOutput, portInner, serverWait, dbDir, actionType+dumpsName,actionType); StoreFile.main(gumOutput, portDumps, serverWait, dbDir, actionType+dumpsName,actionType);
break; break;
case "COMP": case "COMP":
CalculatePairs.main(serverWait, dbDir, actionType+dumpsName, portInner, pairsPath+actionType, pjName+actionType); CalculatePairs.main(serverWait, dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType);
ImportPairs2DB.main(pairsPath+actionType, portInner, serverWait, dbDir,datasetPath); ImportPairs2DB.main(pairsPath+actionType, portInner, serverWait, dbDir,datasetPath);
break;
AkkaTreeLoader.main(portInner, serverWait, dbDir, pjName +actionType+".csv.rdb" , port, actionType+dumpsName); case "AKKA":
AkkaTreeLoader.main(portInner, serverWait, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,cursor);
break;
case "LEVEL1": case "LEVEL1":
level1(portInner, serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput); level1(portInner, serverWait, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput);
break; break;
//CALC python abstractPatch.py to from cluster folder //CALC python abstractPatch.py to from cluster folder
case "LEVEL2": case "LEVEL2":
level2(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); level2(serverWait, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
break; break;
//CALC via python //CALC via python
case "LEVEL3": case "LEVEL3":
level3(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); level3(serverWait, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
break; break;
case "ALL": case "ALL":
// TestHunkParser.main(gumInput, gumOutput, numOfWorkers, pjName); // TestHunkParser.main(gumInput, gumOutput, numOfWorkers, pjName);
StoreFile.main(gumOutput, portInner, serverWait, dbDir, actionType+dumpsName,actionType); // StoreFile.main(gumOutput, portInner, serverWait, dbDir, actionType+dumpsName,actionType);
level1(portInner, serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput); // level1(portInner, serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput);
level2(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); // level2(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
level3(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); // level3(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
break; break;
case "EXTRACTPATTERN": case "EXTRACTPATTERN":
PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,port,pythonPath,datasetPath,pjName,dbNo,actionType,threshold); PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,portDumps,pythonPath,datasetPath,pjName,dbNo,actionType,threshold);
break; break;
case "GETPATTERN": case "GETPATTERN":
PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,port,pythonPath,datasetPath,pjName,dbNo,actionType,threshold); PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,portDumps,pythonPath,datasetPath,pjName,dbNo,actionType,threshold);
break; break;
} }
} catch (Exception e) { } catch (Exception e) {
src/main/java/edu/lu/uni/serval/PatternExtractor.java → src/main/java/edu/lu/uni/serval/fixminer/PatternExtractor.java
+13 -13
View File
@@ -1,19 +1,8 @@
package edu.lu.uni.serval; package edu.lu.uni.serval.fixminer;
import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext; import com.github.gumtreediff.tree.TreeContext;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import edu.lu.uni.serval.FixPattern.utils.Checker; import edu.lu.uni.serval.FixPattern.utils.Checker;
import edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -21,7 +10,18 @@ import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig; import redis.clients.jedis.JedisPoolConfig;
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.getASTTree; import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.getASTTree;
/** /**
* Created by anilkoyuncu on 02/08/2018. * Created by anilkoyuncu on 02/08/2018.
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/AkkaTreeLoader.java
+89 -25
View File
@@ -1,11 +1,10 @@
package edu.lu.uni.serval.FixPatternParser.cluster; package edu.lu.uni.serval.fixminer.cluster;
import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.Tree;
import com.github.gumtreediff.tree.TreeContext; import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap; import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap;
import edu.lu.uni.serval.FixPatternParser.violations.CallShell;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import org.junit.Assert;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import redis.clients.jedis.*; import redis.clients.jedis.*;
@@ -16,7 +15,8 @@ import java.util.*;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.junit.Assert;
import static edu.lu.uni.serval.fixminer.cluster.akka.AkkaTreeParser.akkaCompare;
/** /**
* Created by anilkoyuncu on 19/03/2018. * Created by anilkoyuncu on 19/03/2018.
@@ -71,7 +71,7 @@ public class AkkaTreeLoader {
private static Consumer<String> consumer = Assert::assertNotNull; private static Consumer<String> consumer = Assert::assertNotNull;
public static void main(String portInner,String serverWait,String dbDir,String chunkName,String port, String dumpsName) throws Exception { public static void main(String portInner, String serverWait, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, String cursor) throws Exception {
String parameters = String.format("\nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s \ndumpsName %s",portInner,serverWait,chunkName,dbDir,dumpsName); String parameters = String.format("\nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s \ndumpsName %s",portInner,serverWait,chunkName,dbDir,dumpsName);
@@ -81,57 +81,115 @@ public class AkkaTreeLoader {
CallShell cs = new CallShell(); CallShell cs = new CallShell();
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
String cmd1 = String.format(cmd, dbDir,dumpsName,Integer.valueOf(port)); String cmd1 = String.format(cmd, dbDir,dumpsName,Integer.valueOf(port));
//
cs.runShell(cmd1,serverWait); cs.runShell(cmd1,serverWait);
String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
String cmd2 = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner)); String cmd2 = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner));
log.info(cmd1);
log.info(cmd2);
//
cs.runShell(cmd2,serverWait); cs.runShell(cmd2,serverWait);
JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000); JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000);
// Jedis jedis = new Jedis(new URI("redis://localhost:"+port));
// while (!jedis.ping().equals("PONG")){
// log.info("wait");
// }
//
// jedis = new Jedis(new URI("redis://localhost:"+portInner));
// while (!jedis.ping().equals("PONG")){
// log.info("wait");
// }
comparePairs(innerPool,outerPool);
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; // String pairsIndexFile = pairsPath + "/"+ chunkName;
String stopServer1 = String.format(stopServer,Integer.valueOf(portInner)); // pairsIndexFile = pairsIndexFile.replace("csv.rdb","index");
//
//
// Pattern pattern = Pattern.compile(",");
// String csvFile = pairsIndexFile;
// try {
// try (BufferedReader in = new BufferedReader(new FileReader(csvFile));){
// Map<String,String> namefreq = in
// .lines()
// // .skip(1)
// .map(x -> pattern.split(x))
// // .filter(x -> x[4].equals("CA") && x[3].equals("F"))
// .collect(HashMap::new, (map, x) ->
// map.put(x[0], x[1]),
// Map::putAll);
// // namefreq.forEach((k, v) -> System.out.println(k + " => " + v));
//
// Jedis inner = null;
// try {
// inner = outerPool.getResource();
//
// for (Map.Entry<String, String> entry : namefreq.entrySet()) {
// String key = entry.getKey();
// String value = entry.getValue();
// inner.select(1);
// inner.set(key,value);
// // ...
// }
//
//
// }finally {
// if (inner != null) {
// inner.close();
// }
// }
//
//
// }
// } catch (IOException e) {
// e.printStackTrace();
// }
cs.runShell(stopServer1,serverWait);
stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
String stopServer2 = String.format(stopServer,Integer.valueOf(port));
cs.runShell(stopServer2,serverWait);
// comparePairs(innerPool,outerPool);
akkaCompare(innerPool,outerPool,numOfWorkers,cursor);
// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
// String stopServer1 = String.format(stopServer,Integer.valueOf(portInner));
//
// cs.runShell(stopServer1,serverWait);
// stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
// String stopServer2 = String.format(stopServer,Integer.valueOf(port));
//
// cs.runShell(stopServer2,serverWait);
} }
public static void comparePairs(JedisPool innerPool,JedisPool outerPool){ public static void comparePairs(JedisPool innerPool,JedisPool outerPool){
ScanResult<String> scan; ScanResult<String> scan;
try (Jedis inner = innerPool.getResource()) { try (Jedis inner = innerPool.getResource()) {
while (inner.ping()== "PONG"){ while (!inner.ping().equals("PONG")){
log.info("wait"); log.info("wait");
} }
ScanParams sc = new ScanParams(); ScanParams sc = new ScanParams();
//150000000 //150000000
sc.count(150000000); log.info("Scanning ");
sc.count(250000000);
sc.match("pair_[0-9]*"); sc.match("pair_[0-9]*");
scan = inner.scan("0", sc); scan = inner.scan("0", sc);
int size = scan.getResult().size(); int size = scan.getResult().size();
log.info("Scanning " + String.valueOf(size)); log.info("Scanned " + String.valueOf(size));
} }
List<String> result = scan.getResult(); List<String> result = scan.getResult();
log.info("Getting results");
result result
.parallelStream() .parallelStream()
.forEach(m -> .forEach(m ->
{ {
Compare compare = new Compare(); Compare compare = new Compare();
compare.coreCompare(m, innerPool, outerPool); compare.coreCompare(m, innerPool, outerPool);
@@ -165,14 +223,20 @@ public class AkkaTreeLoader {
Jedis inner = null; Jedis inner = null;
try { try {
inner = outerPool.getResource(); inner = outerPool.getResource();
String s = inner.get(fn); while (!inner.ping().equals("PONG")){
log.info("wait");
}
inner.select(1);
String dist2load = inner.get(fn);
inner.select(0);
String s = inner.get(dist2load);
HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s); HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s);
ITree parent = null; ITree parent = null;
ITree children =null; ITree children =null;
TreeContext tc = new TreeContext(); TreeContext tc = new TreeContext();
tree = getASTTree(actionSet, parent, children,tc); tree = getASTTree(actionSet, parent, children,tc);
// tree.setParent(null); tree.setParent(null);
tc.validate(); tc.validate();
} catch (IOException e) { } catch (IOException e) {
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java
+16 -6
View File
@@ -1,6 +1,5 @@
package edu.lu.uni.serval.FixPatternParser.cluster; package edu.lu.uni.serval.fixminer.cluster;
import edu.lu.uni.serval.FixPatternParser.violations.CallShell;
import edu.lu.uni.serval.utils.FileHelper; import edu.lu.uni.serval.utils.FileHelper;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -14,7 +13,7 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel; import java.nio.channels.FileChannel;
import java.util.List; import java.util.List;
import static edu.lu.uni.serval.FixPatternParser.cluster.TreeLoaderClusterL1.poolConfig; import static edu.lu.uni.serval.fixminer.cluster.TreeLoaderClusterL1.poolConfig;
/** /**
* Created by anilkoyuncu on 05/04/2018. * Created by anilkoyuncu on 05/04/2018.
@@ -63,32 +62,42 @@ public class CalculatePairs {
try { try {
// FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv"); // FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv");
// DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); // DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos));
//
FileOutputStream fosIndex = new FileOutputStream(outputPath + "/" +pjName+".index");
DataOutputStream outStreamIndex = new DataOutputStream(new BufferedOutputStream(fosIndex));
//
// //
// //
// //
// for (int i = 0; i < result.size(); i++) { // for (int i = 0; i < result.size(); i++) {
// line = String.valueOf(i) +"," + result.get(i)+"\n";
// outStreamIndex.write(line.getBytes());
//
// for (int j = i + 1; j < result.size(); j++) { // for (int j = i + 1; j < result.size(); j++) {
// //
// //
// //
// line = String.valueOf(i) +"," + String.valueOf(j) + "," + result.get(i) + "," + result.get(j)+"\n"; // line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n";
// outStream.write(line.getBytes()); // outStream.write(line.getBytes());
// //
// } // }
// } // }
// outStream.close(); // outStream.close();
// outStreamIndex.close();
int fileCounter = 0; int fileCounter = 0;
FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +pjName +String.valueOf(fileCounter)+".txt", "rw").getChannel(); FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +pjName +String.valueOf(fileCounter)+".txt", "rw").getChannel();
int maxSize = 500*500000; int maxSize = 500*1000000;
ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, maxSize); ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, maxSize);
for (int i = 0; i < result.size(); i++) { for (int i = 0; i < result.size(); i++) {
line = String.valueOf(i) +"," + result.get(i)+"\n";
outStreamIndex.write(line.getBytes());
for (int j = i + 1; j < result.size(); j++) { for (int j = i + 1; j < result.size(); j++) {
line = String.valueOf(i) +"\t" + String.valueOf(j) + "\t" + result.get(i) + "\t" + result.get(j)+"\n"; line = String.valueOf(i) +"\t" + String.valueOf(j)+"\n"; // + "\t" + result.get(i) + "\t" + result.get(j)+"\n";
buf = line.getBytes(); buf = line.getBytes();
if(wrBuf.remaining() > 500) { if(wrBuf.remaining() > 500) {
wrBuf.put(buf); wrBuf.put(buf);
@@ -104,6 +113,7 @@ public class CalculatePairs {
} }
} }
outStreamIndex.close();
rwChannel.close(); rwChannel.close();
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
e.printStackTrace(); e.printStackTrace();
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/CallShell.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java
+4 -2
View File
@@ -1,9 +1,11 @@
package edu.lu.uni.serval.FixPatternParser.violations; package edu.lu.uni.serval.fixminer.cluster;
/** /**
* Created by anilkoyuncu on 17/04/2018. * Created by anilkoyuncu on 17/04/2018.
*/ */
import java.io.*;
import java.io.BufferedReader;
import java.io.InputStreamReader;
public class CallShell { public class CallShell {
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/Compare.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java
+12 -10
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.FixPatternParser.cluster; package edu.lu.uni.serval.fixminer.cluster;
import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.ActionGenerator;
import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.actions.model.Action;
@@ -13,7 +13,7 @@ import redis.clients.jedis.JedisPool;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.getSimpliedTree; import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.getSimpliedTree;
/** /**
* Created by anilkoyuncu on 03/04/2018. * Created by anilkoyuncu on 03/04/2018.
@@ -23,7 +23,7 @@ public class Compare {
private Logger log = LoggerFactory.getLogger(Compare.class); private Logger log = LoggerFactory.getLogger(Compare.class);
public void coreCompare(String name , JedisPool innerPool, JedisPool outerPool) { public void coreCompare(String name, JedisPool innerPool, JedisPool outerPool) {
Map<String, String> resultMap; Map<String, String> resultMap;
Jedis jedis = null; Jedis jedis = null;
@@ -32,19 +32,21 @@ public class Compare {
try { try {
jedis = innerPool.getResource(); jedis = innerPool.getResource();
resultMap = jedis.hgetAll(name); // resultMap = jedis.hgetAll(name);
String[] split = name.split("_"); String[] split = name.split("_");
String i = split[1]; String i = split[1];
String j = split[2]; String j = split[2];
String firstValue = resultMap.get("0");
String secondValue = resultMap.get("1");
oldTree = getSimpliedTree(firstValue,outerPool);
newTree = getSimpliedTree(secondValue,outerPool); // String firstValue = resultMap.get("0");
// String secondValue = resultMap.get("1");
oldTree = getSimpliedTree(i,outerPool);
newTree = getSimpliedTree(j,outerPool);
Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
m.match(); m.match();
@@ -63,7 +65,7 @@ public class Compare {
String editDistance = String.valueOf(actions.size()); String editDistance = String.valueOf(actions.size());
String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; String result = i + "," + j + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
@@ -77,7 +79,7 @@ public class Compare {
} }
jedis.select(0);
jedis.del("pair_" + (String.valueOf(i)) + "_" + String.valueOf(j)); jedis.del("pair_" + (String.valueOf(i)) + "_" + String.valueOf(j));
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/ImportPairs2DB.java
+3 -5
View File
@@ -1,18 +1,16 @@
package edu.lu.uni.serval.FixPatternParser.cluster; package edu.lu.uni.serval.fixminer.cluster;
import edu.lu.uni.serval.FixPatternParser.violations.CallShell;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.File; import java.io.File;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
//import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; //import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis;
//import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedisWait; //import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedisWait;
/** /**
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoader.java
+1 -5
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.FixPatternParser.violations; package edu.lu.uni.serval.fixminer.cluster;
import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.ActionGenerator;
import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.actions.model.*;
@@ -14,11 +14,7 @@ import org.slf4j.LoggerFactory;
import redis.clients.jedis.*; import redis.clients.jedis.*;
import java.io.*; import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.sql.Timestamp;
import java.time.Duration; import java.time.Duration;
import java.util.*; import java.util.*;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster.java
+3 -3
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.FixPatternParser.violations; package edu.lu.uni.serval.fixminer.cluster;
import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.ActionGenerator;
import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.actions.model.*;
@@ -33,14 +33,14 @@ public class MultiThreadTreeLoaderCluster {
CallShell cs = new CallShell(); CallShell cs = new CallShell();
String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner)); cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner));
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000"); // edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000");
cs.runShell(cmd1,serverWait); cs.runShell(cmd1,serverWait);
String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port)); cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port));
cs.runShell(cmd2,serverWait); cs.runShell(cmd2,serverWait);
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000"); // edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000");
String cmd3; String cmd3;
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster3.java
+23 -37
View File
@@ -1,41 +1,25 @@
package edu.lu.uni.serval.FixPatternParser.violations; package edu.lu.uni.serval.fixminer.cluster;
import static edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader.getKeysByValue;
import static edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster.fromString;
import java.io.BufferedOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.utils.FileHelper;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.similarity.JaroWinklerDistance; import org.apache.commons.text.similarity.JaroWinklerDistance;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import redis.clients.jedis.*;
import com.github.gumtreediff.tree.ITree; import java.io.*;
import com.github.gumtreediff.tree.TreeContext; import java.time.Duration;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap; import static edu.lu.uni.serval.fixminer.cluster.MultiThreadTreeLoader.getKeysByValue;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import static edu.lu.uni.serval.fixminer.cluster.MultiThreadTreeLoaderCluster.fromString;
import edu.lu.uni.serval.utils.FileHelper;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
/** /**
* Created by anilkoyuncu on 19/03/2018. * Created by anilkoyuncu on 19/03/2018.
@@ -52,12 +36,12 @@ public class MultiThreadTreeLoaderCluster3 {
CallShell cs = new CallShell(); CallShell cs = new CallShell();
String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner)); cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner));
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000"); // edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000");
cs.runShell(cmd1,serverWait); cs.runShell(cmd1,serverWait);
String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port)); cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port));
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000"); // edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000");
cs.runShell(cmd2,serverWait); cs.runShell(cmd2,serverWait);
String cmd3; String cmd3;
@@ -95,7 +79,7 @@ public class MultiThreadTreeLoaderCluster3 {
if (size == 0) { if (size == 0) {
String comd = String.format(cmd3, f.getPath(), portInner); String comd = String.format(cmd3, f.getPath(), portInner);
cs.runShell(comd); cs.runShell(comd);
// edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster. // edu.lu.uni.serval.fixminer.cluster.MultiThreadTreeLoaderCluster.
// loadRedis(comd); // loadRedis(comd);
scan = jedis.scan("0", sc); scan = jedis.scan("0", sc);
@@ -569,20 +553,22 @@ public class MultiThreadTreeLoaderCluster3 {
CharSequence[] oldSequences = oldTokens.toArray(new CharSequence[oldTokens.size()]); CharSequence[] oldSequences = oldTokens.toArray(new CharSequence[oldTokens.size()]);
CharSequence[] newSequences = newTokens.toArray(new CharSequence[newTokens.size()]); CharSequence[] newSequences = newTokens.toArray(new CharSequence[newTokens.size()]);
JaroWinklerDistance jwd = new JaroWinklerDistance(); JaroWinklerDistance jwd = new JaroWinklerDistance();
LevenshteinDistance ld = new LevenshteinDistance(); // LevenshteinDistance ld = new LevenshteinDistance();
Double overallSimi = Double.valueOf(1); Double overallSimi = Double.valueOf(0);
if(oldSequences.length > 0 && (oldSequences.length == newSequences.length)){ if(oldSequences.length > 0 && (oldSequences.length == newSequences.length)){
for (int idx = 0; idx < newSequences.length; idx++) { for (int idx = 0; idx < newSequences.length; idx++) {
Double simi = jwd.apply(newSequences[idx], oldSequences[idx]); Double simi = jwd.apply(newSequences[idx], oldSequences[idx]);
overallSimi = simi * overallSimi; overallSimi = simi + overallSimi;
} }
overallSimi = overallSimi / oldSequences.length;
}else{ }else{
overallSimi = Double.valueOf(0); overallSimi = Double.valueOf(0);
// if(oldSequences.length != 0) { // if(oldSequences.length != 0) {
// log.info("ERROR"); // log.info("ERROR");
// } // }
} }
int retval = Double.compare(overallSimi, Double.valueOf(0.8)); int retval = Double.compare(overallSimi, Double.valueOf(0.8));
if(retval >= 0){ if(retval >= 0){
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/RunnableCompare.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java
+1 -2
View File
@@ -1,6 +1,5 @@
package edu.lu.uni.serval.FixPatternParser.violations; package edu.lu.uni.serval.fixminer.cluster;
import edu.lu.uni.serval.FixPatternParser.cluster.Compare;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
public class RunnableCompare implements Runnable { public class RunnableCompare implements Runnable {
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreFile.java
+2 -7
View File
@@ -1,14 +1,10 @@
package edu.lu.uni.serval.FixPatternParser.cluster; package edu.lu.uni.serval.fixminer.cluster;
import edu.lu.uni.serval.FixPatternParser.violations.CallShell;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import org.apache.commons.lang3.ArrayUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.ScanResult;
import java.io.*; import java.io.*;
import java.util.ArrayList; import java.util.ArrayList;
@@ -18,8 +14,7 @@ import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; import static edu.lu.uni.serval.fixminer.cluster.TreeLoaderClusterL1.poolConfig;
import static edu.lu.uni.serval.FixPatternParser.cluster.TreeLoaderClusterL1.poolConfig;
/** /**
* Created by anilkoyuncu on 03/04/2018. * Created by anilkoyuncu on 03/04/2018.
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeActor.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeActor.java
+17 -24
View File
@@ -1,17 +1,19 @@
package edu.lu.uni.serval.FixPatternParser.violations; package edu.lu.uni.serval.fixminer.cluster;
import akka.actor.ActorRef; import akka.actor.ActorRef;
import akka.actor.Props; import akka.actor.Props;
import akka.actor.UntypedActor; import akka.actor.UntypedActor;
import akka.japi.Creator; import akka.japi.Creator;
import akka.routing.RoundRobinPool; import akka.routing.RoundRobinPool;
import edu.lu.uni.serval.fixminer.cluster.akka.TreeMessage;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.util.List; import java.util.List;
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis;
public class TreeActor extends UntypedActor { public class TreeActor extends UntypedActor {
@@ -21,20 +23,14 @@ public class TreeActor extends UntypedActor {
private final int numberOfWorkers; private final int numberOfWorkers;
private int counter = 0; private int counter = 0;
private String innerPort;
private String dbDir;
private String serverWait;
public TreeActor(int numberOfWorkers,String dbDir,String innerPort,String serverWait) { public TreeActor(int numberOfWorkers) {
mineRouter = this.getContext().actorOf(new RoundRobinPool(numberOfWorkers) mineRouter = this.getContext().actorOf(new RoundRobinPool(numberOfWorkers)
.props(TreeWorker.props()), "tree-router"); .props(TreeWorker.props()), "tree-router");
this.numberOfWorkers = numberOfWorkers; this.numberOfWorkers = numberOfWorkers;
this.innerPort = innerPort;
this.dbDir = dbDir;
this.serverWait = serverWait;
} }
public static Props props(final int numberOfWorkers, final String dbDir,final String innerPort, final String serverWait) { public static Props props(final int numberOfWorkers) {
return Props.create(new Creator<TreeActor>() { return Props.create(new Creator<TreeActor>() {
@@ -42,7 +38,7 @@ public class TreeActor extends UntypedActor {
@Override @Override
public TreeActor create() throws Exception { public TreeActor create() throws Exception {
return new TreeActor(numberOfWorkers,dbDir,innerPort,serverWait); return new TreeActor(numberOfWorkers);
} }
}); });
@@ -51,14 +47,13 @@ public class TreeActor extends UntypedActor {
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
@Override @Override
public void onReceive(Object message) throws Exception { public void onReceive(Object message) throws Exception {
if (message instanceof WorkMessage) { if (message instanceof TreeMessage) {
List<String> files = ((WorkMessage) message).getMsgFiles(); List<String> pairs = ((TreeMessage) message).getName();
String innerPort = ((WorkMessage) message).getInnerPort(); JedisPool innerPool = ((TreeMessage) message).getInnerPool();
String inputPath = ((WorkMessage) message).getInputPath(); JedisPool outerPool = ((TreeMessage) message).getOuterPool();
String dbDir = ((WorkMessage) message).getDbDir();
String serverWait = ((WorkMessage) message).getServerWait();
int size = files.size();
int size = pairs.size();
int average = size / numberOfWorkers; int average = size / numberOfWorkers;
int reminder = size % numberOfWorkers; int reminder = size % numberOfWorkers;
int counter = 0; int counter = 0;
@@ -68,8 +63,8 @@ public class TreeActor extends UntypedActor {
if (counter < reminder) counter ++; if (counter < reminder) counter ++;
int toIndex = (i + 1) * average + counter; int toIndex = (i + 1) * average + counter;
List<String> filesOfWorkers = files.subList(fromIndex, toIndex); List<String> pairsOfWorkers = pairs.subList(fromIndex, toIndex);
final WorkMessage workMsg = new WorkMessage(i + 1, filesOfWorkers,innerPort,inputPath,dbDir,serverWait); final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool);
mineRouter.tell(workMsg, getSelf()); mineRouter.tell(workMsg, getSelf());
logger.info("Assign a task to worker #" + (i + 1) + "..."); logger.info("Assign a task to worker #" + (i + 1) + "...");
} }
@@ -81,9 +76,7 @@ public class TreeActor extends UntypedActor {
this.getContext().stop(mineRouter); this.getContext().stop(mineRouter);
this.getContext().stop(getSelf()); this.getContext().stop(getSelf());
this.getContext().system().shutdown(); this.getContext().system().shutdown();
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
stopServer = String.format(stopServer,Integer.valueOf(innerPort));
loadRedis(stopServer,serverWait);
} }
} else { } else {
unhandled(message); unhandled(message);
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeLoaderClusterL1.java
+1 -2
View File
@@ -1,6 +1,5 @@
package edu.lu.uni.serval.FixPatternParser.cluster; package edu.lu.uni.serval.fixminer.cluster;
import edu.lu.uni.serval.FixPatternParser.violations.CallShell;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import redis.clients.jedis.*; import redis.clients.jedis.*;
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeWorker.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeWorker.java
+89 -61
View File
@@ -1,24 +1,51 @@
package edu.lu.uni.serval.FixPatternParser.violations; package edu.lu.uni.serval.fixminer.cluster;
import akka.actor.Props; import akka.actor.Props;
import akka.actor.UntypedActor; import akka.actor.UntypedActor;
import akka.japi.Creator; import akka.japi.Creator;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.FixPatternParser.cluster.Compare; import edu.lu.uni.serval.fixminer.cluster.akka.TreeMessage;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig; import redis.clients.jedis.JedisPoolConfig;
import java.time.Duration; import java.time.Duration;
import java.util.List; import java.util.List;
import java.util.concurrent.*;
import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.poolConfig;
public class TreeWorker extends UntypedActor { public class TreeWorker extends UntypedActor {
private static Logger log = LoggerFactory.getLogger(TreeWorker.class); private static Logger log = LoggerFactory.getLogger(TreeWorker.class);
// private JedisPool innerPool;
// private JedisPool outerPool;
//
// public TreeWorker(String innerPort,String outerPort) {
//// this.innerPool = innerPool;
//// this.outerPool = outerPool;
// this.outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(outerPort),20000000);
// this.innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(innerPort),20000000);
//
//
// }
//
// public static Props props(final String innerPort,final String outerPort) {
// return Props.create(new Creator<TreeWorker>() {
//
// private static final long serialVersionUID = -7615153844097275009L;
//
// @Override
// public TreeWorker create() throws Exception {
// return new TreeWorker(innerPort,outerPort);
// }
//
// });
// }
public TreeWorker() { public TreeWorker() {
} }
@@ -35,47 +62,48 @@ public class TreeWorker extends UntypedActor {
}); });
} }
@Override @Override
public void onReceive(Object message) throws Exception { public void onReceive(Object message) throws Exception {
if(message instanceof edu.lu.uni.serval.FixPatternParser.violations.WorkMessage) { if(message instanceof TreeMessage) {
// if (message instanceof edu.lu.uni.serval.MultipleThreadsParser.WorkMessage) { // if (message instanceof edu.lu.uni.serval.MultipleThreadsParser.WorkMessage) {
edu.lu.uni.serval.FixPatternParser.violations.WorkMessage msg = (WorkMessage) message; TreeMessage msg = (TreeMessage) message;
List<String> files = msg.getMsgFiles(); List<String> files = msg.getName();
String innerPort = msg.getInnerPort(); JedisPool innerPool = msg.getInnerPool();
String inputPath = msg.getInputPath(); JedisPool outerPool = msg.getOuterPool();
String dbDir = msg.getDbDir();
String serverWait = msg.getServerWait();
int id = msg.getId(); int id = msg.getId();
int counter = new Object() { // int counter = new Object() {
int counter = 0; int counter = 0;
// //
// for (String name : files) for (String name : files)
{ {
files.stream(). // files.stream().
parallel(). // parallel().
peek(x -> counter++). // peek(x -> counter++).
forEach(m -> // forEach(m ->
{ // {
Compare compare = new Compare(); // Compare compare = new Compare();
// compare.coreCompare(m, inputPath, innerPort); // compare.coreCompare(m, innerPool, outerPool);
} // }
); // );
} // }
}.counter; // }.counter;
// //
// final ExecutorService executor = Executors.newFixedThreadPool(20); final ExecutorService executor = Executors.newFixedThreadPool(20);
// // schedule the work // // schedule the work
// final Future<?> future = executor.submit(new RunnableCompare(name, inputPath, innerPort, new Compare(poolConfig))); final Future<?> future = executor.submit(new RunnableCompare(name, innerPool, outerPool, new Compare()));
// try { try {
// wait for task to complete // wait for task to complete
// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS);
// Compare compare = new Compare(poolConfig); Compare compare = new Compare();
// compare.coreCompare(name, inputPath, innerPort); compare.coreCompare(name, innerPool, outerPool);
// counter++; counter++;
// nullDiffEntry += parser.nullMatchedDiffEntry; // nullDiffEntry += parser.nullMatchedDiffEntry;
// nullMappingGumTreeResults += parser.nullMappingGumTreeResult; // nullMappingGumTreeResults += parser.nullMappingGumTreeResult;
// pureDeletion += parser.pureDeletions; // pureDeletion += parser.pureDeletions;
@@ -115,22 +143,22 @@ public class TreeWorker extends UntypedActor {
// testingInfo.setLength(0); // testingInfo.setLength(0);
// } // }
// } // }
// } catch (TimeoutException e) { } catch (TimeoutException e) {
// future.cancel(true); future.cancel(true);
////// timeouts += countAlarms(positionFile, "#Timeout:"); ////// timeouts += countAlarms(positionFile, "#Timeout:");
// System.err.println("#Timeout: " + name); // System.err.println("#Timeout: " + name);
// } catch (InterruptedException e) { } catch (InterruptedException e) {
////// timeouts += countAlarms(positionFile, "#TimeInterrupted:"); ////// timeouts += countAlarms(positionFile, "#TimeInterrupted:");
//// System.err.println("#TimeInterrupted: " + revFile.getName()); //// System.err.println("#TimeInterrupted: " + revFile.getName());
// e.printStackTrace(); e.printStackTrace();
// } catch (ExecutionException e) { } catch (ExecutionException e) {
////// timeouts += countAlarms(positionFile, "#TimeAborted:"); ////// timeouts += countAlarms(positionFile, "#TimeAborted:");
//// System.err.println("#TimeAborted: " + revFile.getName()); //// System.err.println("#TimeAborted: " + revFile.getName());
// e.printStackTrace(); e.printStackTrace();
// } finally { } finally {
// executor.shutdownNow(); executor.shutdownNow();
// } }
// } }
log.info("bitti"); log.info("bitti");
log.info("Worker #" + id +"finialized parsing " + counter + " files..."); log.info("Worker #" + id +"finialized parsing " + counter + " files...");
@@ -169,24 +197,24 @@ public class TreeWorker extends UntypedActor {
} }
} }
static final JedisPoolConfig poolConfig = buildPoolConfig(); // static final JedisPoolConfig poolConfig = buildPoolConfig();
//
//
private static JedisPoolConfig buildPoolConfig() { // private static JedisPoolConfig buildPoolConfig() {
final JedisPoolConfig poolConfig = new JedisPoolConfig(); // final JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMaxTotal(128); // poolConfig.setMaxTotal(128);
poolConfig.setMaxIdle(128); // poolConfig.setMaxIdle(128);
poolConfig.setMinIdle(16); // poolConfig.setMinIdle(16);
poolConfig.setTestOnBorrow(true); // poolConfig.setTestOnBorrow(true);
poolConfig.setTestOnReturn(true); // poolConfig.setTestOnReturn(true);
poolConfig.setTestWhileIdle(true); // poolConfig.setTestWhileIdle(true);
poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); // poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis());
poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); // poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis());
poolConfig.setNumTestsPerEvictionRun(3); // poolConfig.setNumTestsPerEvictionRun(3);
poolConfig.setBlockWhenExhausted(true); // poolConfig.setBlockWhenExhausted(true);
//
return poolConfig; // return poolConfig;
} // }
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/WorkMessage.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/WorkMessage.java
+1 -3
View File
@@ -1,6 +1,4 @@
package edu.lu.uni.serval.FixPatternParser.violations; package edu.lu.uni.serval.fixminer.cluster;
import edu.lu.uni.serval.MultipleThreadsParser.MessageFile;
import java.util.List; import java.util.List;
src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java
+75
View File
@@ -0,0 +1,75 @@
package edu.lu.uni.serval.fixminer.cluster.akka;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import edu.lu.uni.serval.fixminer.cluster.TreeActor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
import java.util.List;
/**
* Created by anilkoyuncu on 12/09/2018.
*/
public class AkkaTreeParser {
private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class);
public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, String cursor){
final List<String> listOfPairs = getMessages(innerPool,cursor); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
ActorSystem system = null;
ActorRef parsingActor = null;
final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool);
try {
log.info("Akka begins...");
system = ActorSystem.create("Compare-EnhancedDiff-System");
parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor");
parsingActor.tell(msg, ActorRef.noSender());
} catch (Exception e) {
system.shutdown();
e.printStackTrace();
}
}
public static List<String> getMessages(JedisPool innerPool, String cursor){
ScanResult<String> scan;
try (Jedis inner = innerPool.getResource()) {
while (!inner.ping().equals("PONG")){
log.info("wait");
}
ScanParams sc = new ScanParams();
//150000000
log.info("Scanning ");
sc.count(Integer.valueOf(cursor));
sc.match("pair_[0-9]*");
scan = inner.scan("0", sc);
int size = scan.getResult().size();
log.info("Scanned " + String.valueOf(size));
}
List<String> result = scan.getResult();
log.info("Getting results");
return result;
}
}
src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java
+54
View File
@@ -0,0 +1,54 @@
package edu.lu.uni.serval.fixminer.cluster.akka;
import redis.clients.jedis.JedisPool;
import java.util.List;
/**
* Created by anilkoyuncu on 12/09/2018.
*/
public class TreeMessage {
private int id;
private List<String> name;
private JedisPool innerPool;
private JedisPool outerPool;
public TreeMessage(int id, List<String> name, JedisPool innerPool, JedisPool outerPool) {
this.id = id;
this.name = name;
this.innerPool = innerPool;
this.outerPool = outerPool;
}
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public List<String> getName() {
return name;
}
public void setName(List<String> name) {
this.name = name;
}
public JedisPool getInnerPool() {
return innerPool;
}
public void setInnerPool(JedisPool innerPool) {
this.innerPool = innerPool;
}
public JedisPool getOuterPool() {
return outerPool;
}
public void setOuterPool(JedisPool outerPool) {
this.outerPool = outerPool;
}
}