azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

akka version of tree comp

Browse Source
This commit is contained in:
fixminer
2018-09-13 16:18:06 +02:00
parent fc5f5b4089
commit 0d391e1ab9
19 changed files with 432 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pom.xml
+2 -2
View File
@@ -140,8 +140,8 @@
<archive>
<manifest>
<mainClass>
edu.lu.uni.serval.Launcher
</mainClass>
edu.lu.uni.serval.fixminer.Launcher
</mainClass>
</manifest>
</archive>
<descriptorRefs>
src/main/java/edu/lu/uni/serval/Launcher.java → src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+26 -24
View File
@@ -1,10 +1,7 @@
package edu.lu.uni.serval;
package edu.lu.uni.serval.fixminer;
import edu.lu.uni.serval.FixPatternParser.cluster.*;
import edu.lu.uni.serval.FixPatternParser.violations.CallShell;
import edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster;
import edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster3;
import edu.lu.uni.serval.FixPatternParser.violations.TestHunkParser;
import edu.lu.uni.serval.fixminer.cluster.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -24,37 +21,40 @@ public class Launcher {
Properties appProps = new Properties();
String appConfigPath = "/Users/kui.liu/Downloads/app.properties";//args[0];
// String appConfigPath = "/Users/kui.liu/Downloads/app.properties";//args[0];
// String appConfigPath = "/Users/anilkoyuncu/bugStudy/release/code/app.properties";
String appConfigPath = args[0];
appProps.load(new FileInputStream(appConfigPath));
String portInner = appProps.getProperty("portInner","6380");
String serverWait = appProps.getProperty("serverWait", "50000");
String numOfWorkers = appProps.getProperty("numOfWorkers", "10");
String jobType = appProps.getProperty("jobType","ALL");
String port = appProps.getProperty("port","6399");
String portDumps = appProps.getProperty("portDumps","6399");
String pythonPath = appProps.getProperty("pythonPath","/Users/anilkoyuncu/bugStudy/code/python");
String datasetPath = appProps.getProperty("datasetPath","/Users/anilkoyuncu/bugStudy/dataset");
String pjName = appProps.getProperty("pjName","allDataset");
String dbNo = appProps.getProperty("dbNo","0");
String actionType = appProps.getProperty("actionType","ALL");
String threshold = appProps.getProperty("threshold","1");
String cursor = appProps.getProperty("cursor","10000000");
String chunk = appProps.getProperty("chunk","1.txt");
String parameters = String.format("\nportInner %s " +
"\nserverWait %s \nnumOfWorkers %s " +
"\njobType %s \nport %s " +
"\npythonPath %s \ndatasetPath %s" +
"\npjName %s \ndbNo %s \nactionType %s \nthreshold %s"
, portInner, serverWait, numOfWorkers, jobType, port, pythonPath,datasetPath,pjName,dbNo,actionType,threshold);
"\npjName %s \ndbNo %s \nactionType %s \nthreshold %s \ncursor %s"
, portInner, serverWait, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,dbNo,actionType,threshold,cursor);
log.info(parameters);
mainLaunch(portInner, serverWait, numOfWorkers, jobType, port, pythonPath,datasetPath,pjName,dbNo,actionType,threshold);
mainLaunch(portInner, serverWait, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,dbNo,actionType,threshold,cursor,chunk);
}
public static void mainLaunch(String portInner,String serverWait, String numOfWorkers,String jobType,String port, String pythonPath, String datasetPath, String pjName, String dbNo, String actionType,String threshold){
public static void mainLaunch(String portInner, String serverWait, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String dbNo, String actionType, String threshold, String cursor, String chunk){
String dbDir;
@@ -75,38 +75,40 @@ public class Launcher {
TestHunkParser.main(gumInput, gumOutput, numOfWorkers, pjName);
break;
case "CACHE":
StoreFile.main(gumOutput, portInner, serverWait, dbDir, actionType+dumpsName,actionType);
StoreFile.main(gumOutput, portDumps, serverWait, dbDir, actionType+dumpsName,actionType);
break;
case "COMP":
CalculatePairs.main(serverWait, dbDir, actionType+dumpsName, portInner, pairsPath+actionType, pjName+actionType);
CalculatePairs.main(serverWait, dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType);
ImportPairs2DB.main(pairsPath+actionType, portInner, serverWait, dbDir,datasetPath);
AkkaTreeLoader.main(portInner, serverWait, dbDir, pjName +actionType+".csv.rdb" , port, actionType+dumpsName);
break;
case "AKKA":
AkkaTreeLoader.main(portInner, serverWait, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,cursor);
break;
case "LEVEL1":
level1(portInner, serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput);
level1(portInner, serverWait, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput);
break;
//CALC python abstractPatch.py to from cluster folder
case "LEVEL2":
level2(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
level2(serverWait, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
break;
//CALC via python
case "LEVEL3":
level3(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
level3(serverWait, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
break;
case "ALL":
// TestHunkParser.main(gumInput, gumOutput, numOfWorkers, pjName);
StoreFile.main(gumOutput, portInner, serverWait, dbDir, actionType+dumpsName,actionType);
level1(portInner, serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput);
level2(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
level3(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
// StoreFile.main(gumOutput, portInner, serverWait, dbDir, actionType+dumpsName,actionType);
// level1(portInner, serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput);
// level2(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
// level3(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
break;
case "EXTRACTPATTERN":
PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,port,pythonPath,datasetPath,pjName,dbNo,actionType,threshold);
PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,portDumps,pythonPath,datasetPath,pjName,dbNo,actionType,threshold);
break;
case "GETPATTERN":
PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,port,pythonPath,datasetPath,pjName,dbNo,actionType,threshold);
PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,portDumps,pythonPath,datasetPath,pjName,dbNo,actionType,threshold);
break;
}
} catch (Exception e) {
src/main/java/edu/lu/uni/serval/PatternExtractor.java → src/main/java/edu/lu/uni/serval/fixminer/PatternExtractor.java
+13 -13
View File
@@ -1,19 +1,8 @@
package edu.lu.uni.serval;
package edu.lu.uni.serval.fixminer;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import edu.lu.uni.serval.FixPattern.utils.Checker;
import edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -21,7 +10,18 @@ import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.getASTTree;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.getASTTree;
/**
* Created by anilkoyuncu on 02/08/2018.
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/AkkaTreeLoader.java
+89 -25
View File
@@ -1,11 +1,10 @@
package edu.lu.uni.serval.FixPatternParser.cluster;
package edu.lu.uni.serval.fixminer.cluster;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.Tree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap;
import edu.lu.uni.serval.FixPatternParser.violations.CallShell;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.*;
@@ -16,7 +15,8 @@ import java.util.*;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.junit.Assert;
import static edu.lu.uni.serval.fixminer.cluster.akka.AkkaTreeParser.akkaCompare;
/**
* Created by anilkoyuncu on 19/03/2018.
@@ -71,7 +71,7 @@ public class AkkaTreeLoader {
private static Consumer<String> consumer = Assert::assertNotNull;
public static void main(String portInner,String serverWait,String dbDir,String chunkName,String port, String dumpsName) throws Exception {
public static void main(String portInner, String serverWait, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, String cursor) throws Exception {
String parameters = String.format("\nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s \ndumpsName %s",portInner,serverWait,chunkName,dbDir,dumpsName);
@@ -81,57 +81,115 @@ public class AkkaTreeLoader {
CallShell cs = new CallShell();
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
String cmd1 = String.format(cmd, dbDir,dumpsName,Integer.valueOf(port));
//
cs.runShell(cmd1,serverWait);
String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
String cmd2 = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner));
log.info(cmd1);
log.info(cmd2);
//
cs.runShell(cmd2,serverWait);
JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000);
// Jedis jedis = new Jedis(new URI("redis://localhost:"+port));
// while (!jedis.ping().equals("PONG")){
// log.info("wait");
// }
//
// jedis = new Jedis(new URI("redis://localhost:"+portInner));
// while (!jedis.ping().equals("PONG")){
// log.info("wait");
// }
comparePairs(innerPool,outerPool);
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
String stopServer1 = String.format(stopServer,Integer.valueOf(portInner));
// String pairsIndexFile = pairsPath + "/"+ chunkName;
// pairsIndexFile = pairsIndexFile.replace("csv.rdb","index");
//
//
// Pattern pattern = Pattern.compile(",");
// String csvFile = pairsIndexFile;
// try {
// try (BufferedReader in = new BufferedReader(new FileReader(csvFile));){
// Map<String,String> namefreq = in
// .lines()
// // .skip(1)
// .map(x -> pattern.split(x))
// // .filter(x -> x[4].equals("CA") && x[3].equals("F"))
// .collect(HashMap::new, (map, x) ->
// map.put(x[0], x[1]),
// Map::putAll);
// // namefreq.forEach((k, v) -> System.out.println(k + " => " + v));
//
// Jedis inner = null;
// try {
// inner = outerPool.getResource();
//
// for (Map.Entry<String, String> entry : namefreq.entrySet()) {
// String key = entry.getKey();
// String value = entry.getValue();
// inner.select(1);
// inner.set(key,value);
// // ...
// }
//
//
// }finally {
// if (inner != null) {
// inner.close();
// }
// }
//
//
// }
// } catch (IOException e) {
// e.printStackTrace();
// }
cs.runShell(stopServer1,serverWait);
stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
String stopServer2 = String.format(stopServer,Integer.valueOf(port));
cs.runShell(stopServer2,serverWait);
// comparePairs(innerPool,outerPool);
akkaCompare(innerPool,outerPool,numOfWorkers,cursor);
// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
// String stopServer1 = String.format(stopServer,Integer.valueOf(portInner));
//
// cs.runShell(stopServer1,serverWait);
// stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
// String stopServer2 = String.format(stopServer,Integer.valueOf(port));
//
// cs.runShell(stopServer2,serverWait);
}
public static void comparePairs(JedisPool innerPool,JedisPool outerPool){
ScanResult<String> scan;
try (Jedis inner = innerPool.getResource()) {
while (inner.ping()== "PONG"){
while (!inner.ping().equals("PONG")){
log.info("wait");
}
ScanParams sc = new ScanParams();
//150000000
sc.count(150000000);
log.info("Scanning ");
sc.count(250000000);
sc.match("pair_[0-9]*");
scan = inner.scan("0", sc);
int size = scan.getResult().size();
log.info("Scanning " + String.valueOf(size));
log.info("Scanned " + String.valueOf(size));
}
List<String> result = scan.getResult();
log.info("Getting results");
result
.parallelStream()
.forEach(m ->
.parallelStream()
.forEach(m ->
{
Compare compare = new Compare();
compare.coreCompare(m, innerPool, outerPool);
@@ -165,14 +223,20 @@ public class AkkaTreeLoader {
Jedis inner = null;
try {
inner = outerPool.getResource();
String s = inner.get(fn);
while (!inner.ping().equals("PONG")){
log.info("wait");
}
inner.select(1);
String dist2load = inner.get(fn);
inner.select(0);
String s = inner.get(dist2load);
HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s);
ITree parent = null;
ITree children =null;
TreeContext tc = new TreeContext();
tree = getASTTree(actionSet, parent, children,tc);
// tree.setParent(null);
tree.setParent(null);
tc.validate();
} catch (IOException e) {
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/CalculatePairs.java
+16 -6
View File
@@ -1,6 +1,5 @@
package edu.lu.uni.serval.FixPatternParser.cluster;
package edu.lu.uni.serval.fixminer.cluster;
import edu.lu.uni.serval.FixPatternParser.violations.CallShell;
import edu.lu.uni.serval.utils.FileHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -14,7 +13,7 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.List;
import static edu.lu.uni.serval.FixPatternParser.cluster.TreeLoaderClusterL1.poolConfig;
import static edu.lu.uni.serval.fixminer.cluster.TreeLoaderClusterL1.poolConfig;
/**
* Created by anilkoyuncu on 05/04/2018.
@@ -63,32 +62,42 @@ public class CalculatePairs {
try {
// FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv");
// DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos));
//
FileOutputStream fosIndex = new FileOutputStream(outputPath + "/" +pjName+".index");
DataOutputStream outStreamIndex = new DataOutputStream(new BufferedOutputStream(fosIndex));
//
//
//
//
// for (int i = 0; i < result.size(); i++) {
// line = String.valueOf(i) +"," + result.get(i)+"\n";
// outStreamIndex.write(line.getBytes());
//
// for (int j = i + 1; j < result.size(); j++) {
//
//
//
// line = String.valueOf(i) +"," + String.valueOf(j) + "," + result.get(i) + "," + result.get(j)+"\n";
// line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n";
// outStream.write(line.getBytes());
//
// }
// }
// outStream.close();
// outStreamIndex.close();
int fileCounter = 0;
FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +pjName +String.valueOf(fileCounter)+".txt", "rw").getChannel();
int maxSize = 500*500000;
int maxSize = 500*1000000;
ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, maxSize);
for (int i = 0; i < result.size(); i++) {
line = String.valueOf(i) +"," + result.get(i)+"\n";
outStreamIndex.write(line.getBytes());
for (int j = i + 1; j < result.size(); j++) {
line = String.valueOf(i) +"\t" + String.valueOf(j) + "\t" + result.get(i) + "\t" + result.get(j)+"\n";
line = String.valueOf(i) +"\t" + String.valueOf(j)+"\n"; // + "\t" + result.get(i) + "\t" + result.get(j)+"\n";
buf = line.getBytes();
if(wrBuf.remaining() > 500) {
wrBuf.put(buf);
@@ -104,6 +113,7 @@ public class CalculatePairs {
}
}
outStreamIndex.close();
rwChannel.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/CallShell.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/CallShell.java
+4 -2
View File
@@ -1,9 +1,11 @@
package edu.lu.uni.serval.FixPatternParser.violations;
package edu.lu.uni.serval.fixminer.cluster;
/**
* Created by anilkoyuncu on 17/04/2018.
*/
import java.io.*;
import java.io.BufferedReader;
import java.io.InputStreamReader;
public class CallShell {
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/Compare.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java
+12 -10
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.FixPatternParser.cluster;
package edu.lu.uni.serval.fixminer.cluster;
import com.github.gumtreediff.actions.ActionGenerator;
import com.github.gumtreediff.actions.model.Action;
@@ -13,7 +13,7 @@ import redis.clients.jedis.JedisPool;
import java.util.List;
import java.util.Map;
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.getSimpliedTree;
import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.getSimpliedTree;
/**
* Created by anilkoyuncu on 03/04/2018.
@@ -23,7 +23,7 @@ public class Compare {
private Logger log = LoggerFactory.getLogger(Compare.class);
public void coreCompare(String name , JedisPool innerPool, JedisPool outerPool) {
public void coreCompare(String name, JedisPool innerPool, JedisPool outerPool) {
Map<String, String> resultMap;
Jedis jedis = null;
@@ -32,19 +32,21 @@ public class Compare {
try {
jedis = innerPool.getResource();
resultMap = jedis.hgetAll(name);
// resultMap = jedis.hgetAll(name);
String[] split = name.split("_");
String i = split[1];
String j = split[2];
String firstValue = resultMap.get("0");
String secondValue = resultMap.get("1");
oldTree = getSimpliedTree(firstValue,outerPool);
newTree = getSimpliedTree(secondValue,outerPool);
// String firstValue = resultMap.get("0");
// String secondValue = resultMap.get("1");
oldTree = getSimpliedTree(i,outerPool);
newTree = getSimpliedTree(j,outerPool);
Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
m.match();
@@ -63,7 +65,7 @@ public class Compare {
String editDistance = String.valueOf(actions.size());
String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
String result = i + "," + j + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
@@ -77,7 +79,7 @@ public class Compare {
}
jedis.select(0);
jedis.del("pair_" + (String.valueOf(i)) + "_" + String.valueOf(j));
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/ImportPairs2DB.java
+3 -5
View File
@@ -1,18 +1,16 @@
package edu.lu.uni.serval.FixPatternParser.cluster;
package edu.lu.uni.serval.fixminer.cluster;
import edu.lu.uni.serval.FixPatternParser.violations.CallShell;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
//import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis;
//import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedisWait;
//import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis;
//import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedisWait;
/**
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoader.java
+1 -5
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.FixPatternParser.violations;
package edu.lu.uni.serval.fixminer.cluster;
import com.github.gumtreediff.actions.ActionGenerator;
import com.github.gumtreediff.actions.model.*;
@@ -14,11 +14,7 @@ import org.slf4j.LoggerFactory;
import redis.clients.jedis.*;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.sql.Timestamp;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.Executors;
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster.java
+3 -3
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.FixPatternParser.violations;
package edu.lu.uni.serval.fixminer.cluster;
import com.github.gumtreediff.actions.ActionGenerator;
import com.github.gumtreediff.actions.model.*;
@@ -33,14 +33,14 @@ public class MultiThreadTreeLoaderCluster {
CallShell cs = new CallShell();
String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner));
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000");
// edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000");
cs.runShell(cmd1,serverWait);
String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port));
cs.runShell(cmd2,serverWait);
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000");
// edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000");
String cmd3;
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/MultiThreadTreeLoaderCluster3.java
+23 -37
View File
@@ -1,41 +1,25 @@
package edu.lu.uni.serval.FixPatternParser.violations;
import static edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader.getKeysByValue;
import static edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster.fromString;
import java.io.BufferedOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
package edu.lu.uni.serval.fixminer.cluster;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.utils.FileHelper;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.similarity.JaroWinklerDistance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.*;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import java.io.*;
import java.time.Duration;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.utils.FileHelper;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
import static edu.lu.uni.serval.fixminer.cluster.MultiThreadTreeLoader.getKeysByValue;
import static edu.lu.uni.serval.fixminer.cluster.MultiThreadTreeLoaderCluster.fromString;
/**
* Created by anilkoyuncu on 19/03/2018.
@@ -52,12 +36,12 @@ public class MultiThreadTreeLoaderCluster3 {
CallShell cs = new CallShell();
String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner));
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000");
// edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000");
cs.runShell(cmd1,serverWait);
String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port));
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000");
// edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000");
cs.runShell(cmd2,serverWait);
String cmd3;
@@ -95,7 +79,7 @@ public class MultiThreadTreeLoaderCluster3 {
if (size == 0) {
String comd = String.format(cmd3, f.getPath(), portInner);
cs.runShell(comd);
// edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster.
// edu.lu.uni.serval.fixminer.cluster.MultiThreadTreeLoaderCluster.
// loadRedis(comd);
scan = jedis.scan("0", sc);
@@ -569,20 +553,22 @@ public class MultiThreadTreeLoaderCluster3 {
CharSequence[] oldSequences = oldTokens.toArray(new CharSequence[oldTokens.size()]);
CharSequence[] newSequences = newTokens.toArray(new CharSequence[newTokens.size()]);
JaroWinklerDistance jwd = new JaroWinklerDistance();
LevenshteinDistance ld = new LevenshteinDistance();
// LevenshteinDistance ld = new LevenshteinDistance();
Double overallSimi = Double.valueOf(1);
Double overallSimi = Double.valueOf(0);
if(oldSequences.length > 0 && (oldSequences.length == newSequences.length)){
for (int idx = 0; idx < newSequences.length; idx++) {
Double simi = jwd.apply(newSequences[idx], oldSequences[idx]);
overallSimi = simi * overallSimi;
overallSimi = simi + overallSimi;
}
overallSimi = overallSimi / oldSequences.length;
}else{
overallSimi = Double.valueOf(0);
// if(oldSequences.length != 0) {
// log.info("ERROR");
// }
}
int retval = Double.compare(overallSimi, Double.valueOf(0.8));
if(retval >= 0){
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/RunnableCompare.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java
+1 -2
View File
@@ -1,6 +1,5 @@
package edu.lu.uni.serval.FixPatternParser.violations;
package edu.lu.uni.serval.fixminer.cluster;
import edu.lu.uni.serval.FixPatternParser.cluster.Compare;
import redis.clients.jedis.JedisPool;
public class RunnableCompare implements Runnable {
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/StoreFile.java
+2 -7
View File
@@ -1,14 +1,10 @@
package edu.lu.uni.serval.FixPatternParser.cluster;
package edu.lu.uni.serval.fixminer.cluster;
import edu.lu.uni.serval.FixPatternParser.violations.CallShell;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import org.apache.commons.lang3.ArrayUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.ScanResult;
import java.io.*;
import java.util.ArrayList;
@@ -18,8 +14,7 @@ import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis;
import static edu.lu.uni.serval.FixPatternParser.cluster.TreeLoaderClusterL1.poolConfig;
import static edu.lu.uni.serval.fixminer.cluster.TreeLoaderClusterL1.poolConfig;
/**
* Created by anilkoyuncu on 03/04/2018.
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeActor.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeActor.java
+17 -24
View File
@@ -1,17 +1,19 @@
package edu.lu.uni.serval.FixPatternParser.violations;
package edu.lu.uni.serval.fixminer.cluster;
import akka.actor.ActorRef;
import akka.actor.Props;
import akka.actor.UntypedActor;
import akka.japi.Creator;
import akka.routing.RoundRobinPool;
import edu.lu.uni.serval.fixminer.cluster.akka.TreeMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import java.util.List;
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis;
import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.loadRedis;
public class TreeActor extends UntypedActor {
@@ -21,20 +23,14 @@ public class TreeActor extends UntypedActor {
private final int numberOfWorkers;
private int counter = 0;
private String innerPort;
private String dbDir;
private String serverWait;
public TreeActor(int numberOfWorkers,String dbDir,String innerPort,String serverWait) {
public TreeActor(int numberOfWorkers) {
mineRouter = this.getContext().actorOf(new RoundRobinPool(numberOfWorkers)
.props(TreeWorker.props()), "tree-router");
this.numberOfWorkers = numberOfWorkers;
this.innerPort = innerPort;
this.dbDir = dbDir;
this.serverWait = serverWait;
}
public static Props props(final int numberOfWorkers, final String dbDir,final String innerPort, final String serverWait) {
public static Props props(final int numberOfWorkers) {
return Props.create(new Creator<TreeActor>() {
@@ -42,7 +38,7 @@ public class TreeActor extends UntypedActor {
@Override
public TreeActor create() throws Exception {
return new TreeActor(numberOfWorkers,dbDir,innerPort,serverWait);
return new TreeActor(numberOfWorkers);
}
});
@@ -51,14 +47,13 @@ public class TreeActor extends UntypedActor {
@SuppressWarnings("deprecation")
@Override
public void onReceive(Object message) throws Exception {
if (message instanceof WorkMessage) {
List<String> files = ((WorkMessage) message).getMsgFiles();
String innerPort = ((WorkMessage) message).getInnerPort();
String inputPath = ((WorkMessage) message).getInputPath();
String dbDir = ((WorkMessage) message).getDbDir();
String serverWait = ((WorkMessage) message).getServerWait();
if (message instanceof TreeMessage) {
List<String> pairs = ((TreeMessage) message).getName();
JedisPool innerPool = ((TreeMessage) message).getInnerPool();
JedisPool outerPool = ((TreeMessage) message).getOuterPool();
int size = files.size();
int size = pairs.size();
int average = size / numberOfWorkers;
int reminder = size % numberOfWorkers;
int counter = 0;
@@ -68,8 +63,8 @@ public class TreeActor extends UntypedActor {
if (counter < reminder) counter ++;
int toIndex = (i + 1) * average + counter;
List<String> filesOfWorkers = files.subList(fromIndex, toIndex);
final WorkMessage workMsg = new WorkMessage(i + 1, filesOfWorkers,innerPort,inputPath,dbDir,serverWait);
List<String> pairsOfWorkers = pairs.subList(fromIndex, toIndex);
final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool);
mineRouter.tell(workMsg, getSelf());
logger.info("Assign a task to worker #" + (i + 1) + "...");
}
@@ -81,9 +76,7 @@ public class TreeActor extends UntypedActor {
this.getContext().stop(mineRouter);
this.getContext().stop(getSelf());
this.getContext().system().shutdown();
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
stopServer = String.format(stopServer,Integer.valueOf(innerPort));
loadRedis(stopServer,serverWait);
}
} else {
unhandled(message);
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeLoaderClusterL1.java
+1 -2
View File
@@ -1,6 +1,5 @@
package edu.lu.uni.serval.FixPatternParser.cluster;
package edu.lu.uni.serval.fixminer.cluster;
import edu.lu.uni.serval.FixPatternParser.violations.CallShell;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.*;
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeWorker.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/TreeWorker.java
+89 -61
View File
@@ -1,24 +1,51 @@
package edu.lu.uni.serval.FixPatternParser.violations;
package edu.lu.uni.serval.fixminer.cluster;
import akka.actor.Props;
import akka.actor.UntypedActor;
import akka.japi.Creator;
import edu.lu.uni.serval.FixPatternParser.cluster.Compare;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.fixminer.cluster.akka.TreeMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.*;
import static edu.lu.uni.serval.fixminer.cluster.AkkaTreeLoader.poolConfig;
public class TreeWorker extends UntypedActor {
private static Logger log = LoggerFactory.getLogger(TreeWorker.class);
// private JedisPool innerPool;
// private JedisPool outerPool;
//
// public TreeWorker(String innerPort,String outerPort) {
//// this.innerPool = innerPool;
//// this.outerPool = outerPool;
// this.outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(outerPort),20000000);
// this.innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(innerPort),20000000);
//
//
// }
//
// public static Props props(final String innerPort,final String outerPort) {
// return Props.create(new Creator<TreeWorker>() {
//
// private static final long serialVersionUID = -7615153844097275009L;
//
// @Override
// public TreeWorker create() throws Exception {
// return new TreeWorker(innerPort,outerPort);
// }
//
// });
// }
public TreeWorker() {
public TreeWorker() {
}
@@ -35,47 +62,48 @@ public class TreeWorker extends UntypedActor {
});
}
@Override
public void onReceive(Object message) throws Exception {
if(message instanceof edu.lu.uni.serval.FixPatternParser.violations.WorkMessage) {
if(message instanceof TreeMessage) {
// if (message instanceof edu.lu.uni.serval.MultipleThreadsParser.WorkMessage) {
edu.lu.uni.serval.FixPatternParser.violations.WorkMessage msg = (WorkMessage) message;
List<String> files = msg.getMsgFiles();
String innerPort = msg.getInnerPort();
String inputPath = msg.getInputPath();
String dbDir = msg.getDbDir();
String serverWait = msg.getServerWait();
TreeMessage msg = (TreeMessage) message;
List<String> files = msg.getName();
JedisPool innerPool = msg.getInnerPool();
JedisPool outerPool = msg.getOuterPool();
int id = msg.getId();
int counter = new Object() {
// int counter = new Object() {
int counter = 0;
//
// for (String name : files)
for (String name : files)
{
files.stream().
parallel().
peek(x -> counter++).
forEach(m ->
{
Compare compare = new Compare();
// compare.coreCompare(m, inputPath, innerPort);
}
);
}
}.counter;
// files.stream().
// parallel().
// peek(x -> counter++).
// forEach(m ->
// {
// Compare compare = new Compare();
// compare.coreCompare(m, innerPool, outerPool);
// }
// );
// }
// }.counter;
//
// final ExecutorService executor = Executors.newFixedThreadPool(20);
final ExecutorService executor = Executors.newFixedThreadPool(20);
// // schedule the work
// final Future<?> future = executor.submit(new RunnableCompare(name, inputPath, innerPort, new Compare(poolConfig)));
// try {
// wait for task to complete
// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS);
// Compare compare = new Compare(poolConfig);
// compare.coreCompare(name, inputPath, innerPort);
// counter++;
final Future<?> future = executor.submit(new RunnableCompare(name, innerPool, outerPool, new Compare()));
try {
// wait for task to complete
future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS);
Compare compare = new Compare();
compare.coreCompare(name, innerPool, outerPool);
counter++;
// nullDiffEntry += parser.nullMatchedDiffEntry;
// nullMappingGumTreeResults += parser.nullMappingGumTreeResult;
// pureDeletion += parser.pureDeletions;
@@ -115,22 +143,22 @@ public class TreeWorker extends UntypedActor {
// testingInfo.setLength(0);
// }
// }
// } catch (TimeoutException e) {
// future.cancel(true);
} catch (TimeoutException e) {
future.cancel(true);
////// timeouts += countAlarms(positionFile, "#Timeout:");
// System.err.println("#Timeout: " + name);
// } catch (InterruptedException e) {
} catch (InterruptedException e) {
////// timeouts += countAlarms(positionFile, "#TimeInterrupted:");
//// System.err.println("#TimeInterrupted: " + revFile.getName());
// e.printStackTrace();
// } catch (ExecutionException e) {
e.printStackTrace();
} catch (ExecutionException e) {
////// timeouts += countAlarms(positionFile, "#TimeAborted:");
//// System.err.println("#TimeAborted: " + revFile.getName());
// e.printStackTrace();
// } finally {
// executor.shutdownNow();
// }
// }
e.printStackTrace();
} finally {
executor.shutdownNow();
}
}
log.info("bitti");
log.info("Worker #" + id +"finialized parsing " + counter + " files...");
@@ -169,24 +197,24 @@ public class TreeWorker extends UntypedActor {
}
}
static final JedisPoolConfig poolConfig = buildPoolConfig();
private static JedisPoolConfig buildPoolConfig() {
final JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMaxTotal(128);
poolConfig.setMaxIdle(128);
poolConfig.setMinIdle(16);
poolConfig.setTestOnBorrow(true);
poolConfig.setTestOnReturn(true);
poolConfig.setTestWhileIdle(true);
poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis());
poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis());
poolConfig.setNumTestsPerEvictionRun(3);
poolConfig.setBlockWhenExhausted(true);
return poolConfig;
}
// static final JedisPoolConfig poolConfig = buildPoolConfig();
//
//
// private static JedisPoolConfig buildPoolConfig() {
// final JedisPoolConfig poolConfig = new JedisPoolConfig();
// poolConfig.setMaxTotal(128);
// poolConfig.setMaxIdle(128);
// poolConfig.setMinIdle(16);
// poolConfig.setTestOnBorrow(true);
// poolConfig.setTestOnReturn(true);
// poolConfig.setTestWhileIdle(true);
// poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis());
// poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis());
// poolConfig.setNumTestsPerEvictionRun(3);
// poolConfig.setBlockWhenExhausted(true);
//
// return poolConfig;
// }
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/WorkMessage.java → src/main/java/edu/lu/uni/serval/fixminer/cluster/WorkMessage.java
+1 -3
View File
@@ -1,6 +1,4 @@
package edu.lu.uni.serval.FixPatternParser.violations;
import edu.lu.uni.serval.MultipleThreadsParser.MessageFile;
package edu.lu.uni.serval.fixminer.cluster;
import java.util.List;
src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java
+75
View File
@@ -0,0 +1,75 @@
package edu.lu.uni.serval.fixminer.cluster.akka;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import edu.lu.uni.serval.fixminer.cluster.TreeActor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
import java.util.List;
/**
* Created by anilkoyuncu on 12/09/2018.
*/
public class AkkaTreeParser {
private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class);
public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, String cursor){
final List<String> listOfPairs = getMessages(innerPool,cursor); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
ActorSystem system = null;
ActorRef parsingActor = null;
final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool);
try {
log.info("Akka begins...");
system = ActorSystem.create("Compare-EnhancedDiff-System");
parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor");
parsingActor.tell(msg, ActorRef.noSender());
} catch (Exception e) {
system.shutdown();
e.printStackTrace();
}
}
public static List<String> getMessages(JedisPool innerPool, String cursor){
ScanResult<String> scan;
try (Jedis inner = innerPool.getResource()) {
while (!inner.ping().equals("PONG")){
log.info("wait");
}
ScanParams sc = new ScanParams();
//150000000
log.info("Scanning ");
sc.count(Integer.valueOf(cursor));
sc.match("pair_[0-9]*");
scan = inner.scan("0", sc);
int size = scan.getResult().size();
log.info("Scanned " + String.valueOf(size));
}
List<String> result = scan.getResult();
log.info("Getting results");
return result;
}
}
src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java
+54
View File
@@ -0,0 +1,54 @@
package edu.lu.uni.serval.fixminer.cluster.akka;
import redis.clients.jedis.JedisPool;
import java.util.List;
/**
* Created by anilkoyuncu on 12/09/2018.
*/
public class TreeMessage {
private int id;
private List<String> name;
private JedisPool innerPool;
private JedisPool outerPool;
public TreeMessage(int id, List<String> name, JedisPool innerPool, JedisPool outerPool) {
this.id = id;
this.name = name;
this.innerPool = innerPool;
this.outerPool = outerPool;
}
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public List<String> getName() {
return name;
}
public void setName(List<String> name) {
this.name = name;
}
public JedisPool getInnerPool() {
return innerPool;
}
public void setInnerPool(JedisPool innerPool) {
this.innerPool = innerPool;
}
public JedisPool getOuterPool() {
return outerPool;
}
public void setOuterPool(JedisPool outerPool) {
this.outerPool = outerPool;
}
}