calculate pairs
This commit is contained in:
Anil Koyuncu
@@ -108,7 +108,7 @@
|
||||
<archive>
|
||||
<manifest>
|
||||
<mainClass>
|
||||
edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader
|
||||
edu.lu.uni.serval.FixPatternParser.cluster.CalculatePairs
|
||||
</mainClass>
|
||||
</manifest>
|
||||
</archive>
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
package edu.lu.uni.serval.FixPatternParser.cluster;
|
||||
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
import com.github.gumtreediff.tree.Tree;
|
||||
import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap;
|
||||
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.*;
|
||||
|
||||
import java.io.*;
|
||||
import java.time.Duration;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Created by anilkoyuncu on 19/03/2018.
|
||||
*/
|
||||
public class AkkaTreeLoader {
|
||||
|
||||
private static class StreamGobbler implements Runnable {
|
||||
private InputStream inputStream;
|
||||
private Consumer<String> consumer;
|
||||
|
||||
public StreamGobbler(InputStream inputStream, Consumer<String> consumer) {
|
||||
this.inputStream = inputStream;
|
||||
this.consumer = consumer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
new BufferedReader(new InputStreamReader(inputStream)).lines()
|
||||
.forEach(consumer);
|
||||
}
|
||||
}
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class);
|
||||
|
||||
|
||||
public static void loadRedis(String cmd,String serverWait){
|
||||
Process process;
|
||||
|
||||
try {
|
||||
// String comd = String.format(cmd, f.getAbsoluteFile());
|
||||
process = Runtime.getRuntime()
|
||||
|
||||
.exec(cmd);
|
||||
|
||||
|
||||
StreamGobbler streamGobbler =
|
||||
new StreamGobbler(process.getInputStream(), System.out::println);
|
||||
Executors.newSingleThreadExecutor().submit(streamGobbler);
|
||||
// int exitCode = process.waitFor();
|
||||
// assert exitCode == 0;
|
||||
Thread.sleep(Integer.valueOf(serverWait));
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
log.info("Load done");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
|
||||
String inputPath;
|
||||
// String outputPath;
|
||||
String port;
|
||||
String portInner;
|
||||
// String pairsCSVPath;
|
||||
String importScript;
|
||||
// String pairsCompletedPath;
|
||||
String serverWait;
|
||||
// String option;
|
||||
String dbDir;
|
||||
String chunkName;
|
||||
String numOfWorkers;
|
||||
if (args.length > 0) {
|
||||
inputPath = args[0];
|
||||
portInner = args[1];
|
||||
serverWait = args[2];
|
||||
// option = args[4];
|
||||
chunkName = args[3];
|
||||
numOfWorkers = args[4];
|
||||
dbDir = args[5];
|
||||
port = args[6];
|
||||
// pairsCSVPath = args[3];
|
||||
// importScript = args[4];
|
||||
// pairsCompletedPath = args[3];
|
||||
} else {
|
||||
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
|
||||
// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/";
|
||||
port = "6399";
|
||||
portInner = "6380";
|
||||
serverWait = "10000";
|
||||
// option = "COMP";
|
||||
// pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test";
|
||||
// importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh";
|
||||
// pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed";
|
||||
chunkName ="chunk3.rdb";
|
||||
dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
|
||||
numOfWorkers = "1";
|
||||
|
||||
}
|
||||
String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \nnumOfWorks %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,numOfWorkers,dbDir);
|
||||
log.info(parameters);
|
||||
|
||||
// if (option.equals("CALC")) {
|
||||
// calculatePairs(inputPath, port);
|
||||
// log.info("Calculate pairs done");
|
||||
// }else {
|
||||
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
|
||||
cmd = String.format(cmd, dbDir,"dumps.rdb",Integer.valueOf(port));
|
||||
loadRedis(cmd,serverWait);
|
||||
|
||||
String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
|
||||
cmd = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner));
|
||||
loadRedis(cmd,serverWait);
|
||||
|
||||
JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
|
||||
JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000);
|
||||
|
||||
|
||||
|
||||
comparePairs(inputPath, innerPool,outerPool, serverWait,chunkName,dbDir,numOfWorkers);
|
||||
|
||||
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
|
||||
stopServer = String.format(stopServer,Integer.valueOf(portInner));
|
||||
loadRedis(stopServer,serverWait);
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static void comparePairs(String inputPath, JedisPool innerPool,JedisPool outerPool,String serverWait,String chunkName, String dbDir,String numOfWorkers){
|
||||
|
||||
|
||||
ScanResult<String> scan;
|
||||
try (Jedis inner = innerPool.getResource()) {
|
||||
while (inner.ping()== "PONG"){
|
||||
log.info("wait");
|
||||
}
|
||||
|
||||
ScanParams sc = new ScanParams();
|
||||
//150000000
|
||||
sc.count(150000000);
|
||||
sc.match("pair_*");
|
||||
|
||||
scan = inner.scan("0", sc);
|
||||
int size = scan.getResult().size();
|
||||
log.info("Scanning " + String.valueOf(size));
|
||||
}
|
||||
List<String> result = scan.getResult();
|
||||
|
||||
|
||||
// ActorSystem system = null;
|
||||
// ActorRef parsingActor = null;
|
||||
// final WorkMessage msg = new WorkMessage(0, result,innerPort,inputPath,dbDir,serverWait);
|
||||
// try {
|
||||
//
|
||||
// log.info("Akka begins...");
|
||||
// system = ActorSystem.create("Tree-System");
|
||||
// parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers),dbDir,innerPort,serverWait), "tree-actor");
|
||||
// parsingActor.tell(msg, ActorRef.noSender());
|
||||
// } catch (Exception e) {
|
||||
// system.shutdown();
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// greeter.tell();
|
||||
result
|
||||
.parallelStream()
|
||||
.forEach(m ->
|
||||
{
|
||||
Compare compare = new Compare();
|
||||
compare.coreCompare(m, innerPool, outerPool);
|
||||
;
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
|
||||
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
/** Read the object from Base64 string. */
|
||||
private static Object fromString( String s ) throws IOException ,
|
||||
ClassNotFoundException {
|
||||
byte [] data = Base64.getDecoder().decode( s );
|
||||
ObjectInputStream ois = new ObjectInputStream(
|
||||
new ByteArrayInputStream( data ) );
|
||||
Object o = ois.readObject();
|
||||
ois.close();
|
||||
return o;
|
||||
}
|
||||
public static ITree getSimpliedTree(String fn,JedisPool outerPool) {
|
||||
|
||||
ITree tree = null;
|
||||
Jedis inner = null;
|
||||
try {
|
||||
inner = outerPool.getResource();
|
||||
String s = inner.get(fn.substring(1));
|
||||
HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s);
|
||||
|
||||
ITree parent = null;
|
||||
ITree children =null;
|
||||
tree = getASTTree(actionSet, parent, children);
|
||||
tree.setParent(null);
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}finally {
|
||||
if (inner != null) {
|
||||
inner.close();
|
||||
}
|
||||
}
|
||||
return tree;
|
||||
|
||||
}
|
||||
|
||||
public static <T, E> List<T> getKeysByValue(Map<T, E> map, E value) {
|
||||
return map.entrySet()
|
||||
.stream()
|
||||
.filter(entry -> Objects.equals(entry.getValue(), value))
|
||||
.map(Map.Entry::getKey)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children){
|
||||
|
||||
int newType = 0;
|
||||
|
||||
String astNodeType = actionSet.getAstNodeType();
|
||||
List<Integer> keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType);
|
||||
|
||||
if(keysByValue.size() != 1){
|
||||
log.error("Birden cok astnodemapmapping");
|
||||
}
|
||||
newType = keysByValue.get(0);
|
||||
if(actionSet.getParent() == null){
|
||||
//root
|
||||
|
||||
parent = new Tree(newType,"");
|
||||
}else{
|
||||
children = new Tree(newType,"");
|
||||
parent.addChild(children);
|
||||
}
|
||||
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
|
||||
if (subActions.size() != 0){
|
||||
for (HierarchicalActionSet subAction : subActions) {
|
||||
|
||||
if(actionSet.getParent() == null){
|
||||
children = parent;
|
||||
}
|
||||
getASTTree(subAction,children,null);
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
return parent;
|
||||
}
|
||||
|
||||
|
||||
static final JedisPoolConfig poolConfig = buildPoolConfig();
|
||||
|
||||
|
||||
private static JedisPoolConfig buildPoolConfig() {
|
||||
final JedisPoolConfig poolConfig = new JedisPoolConfig();
|
||||
poolConfig.setMaxTotal(128);
|
||||
poolConfig.setMaxIdle(128);
|
||||
poolConfig.setMinIdle(16);
|
||||
poolConfig.setTestOnBorrow(false);
|
||||
poolConfig.setTestOnReturn(false);
|
||||
poolConfig.setTestWhileIdle(true);
|
||||
poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis());
|
||||
poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis());
|
||||
// poolConfig.setNumTestsPerEvictionRun(3);
|
||||
poolConfig.setBlockWhenExhausted(true);
|
||||
|
||||
return poolConfig;
|
||||
}
|
||||
|
||||
private static List<edu.lu.uni.serval.MultipleThreadsParser.MessageFile> getMessageFiles(String gumTreeInput) {
|
||||
String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile
|
||||
File revFilesPath = new File(inputPath + "revFiles/");
|
||||
File[] revFiles = revFilesPath.listFiles(); // project folders
|
||||
List<edu.lu.uni.serval.MultipleThreadsParser.MessageFile> msgFiles = new ArrayList<>();
|
||||
if (revFiles.length >= 0) {
|
||||
for (File revFile : revFiles) {
|
||||
// if (revFile.getName().endsWith(".java")) {
|
||||
String fileName = revFile.getName();
|
||||
File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
|
||||
fileName = fileName.replace(".java", ".txt");
|
||||
File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file
|
||||
File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file
|
||||
edu.lu.uni.serval.MultipleThreadsParser.MessageFile msgFile = new edu.lu.uni.serval.MultipleThreadsParser.MessageFile(revFile, prevFile, diffentryFile);
|
||||
msgFile.setPositionFile(positionFile);
|
||||
msgFiles.add(msgFile);
|
||||
// }
|
||||
}
|
||||
|
||||
return msgFiles;
|
||||
}
|
||||
else{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
package edu.lu.uni.serval.FixPatternParser.cluster;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.Jedis;
|
||||
import redis.clients.jedis.JedisPool;
|
||||
import redis.clients.jedis.ScanParams;
|
||||
import redis.clients.jedis.ScanResult;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.List;
|
||||
|
||||
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis;
|
||||
import static edu.lu.uni.serval.FixPatternParser.cluster.TreeLoaderClusterL1.poolConfig;
|
||||
|
||||
/**
|
||||
* Created by anilkoyuncu on 05/04/2018.
|
||||
*/
|
||||
public class CalculatePairs {
|
||||
private static Logger log = LoggerFactory.getLogger(CalculatePairs.class);
|
||||
public static void main(String[] args) {
|
||||
|
||||
|
||||
String inputPath;
|
||||
String port;
|
||||
String portInner;
|
||||
String serverWait;
|
||||
String dbDir;
|
||||
String chunkName;
|
||||
|
||||
if (args.length > 0) {
|
||||
inputPath = args[0];
|
||||
portInner = args[1];
|
||||
serverWait = args[2];
|
||||
chunkName = args[3];
|
||||
|
||||
dbDir = args[5];
|
||||
port = args[6];
|
||||
|
||||
} else {
|
||||
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
|
||||
|
||||
port = "6399";
|
||||
portInner = "6380";
|
||||
serverWait = "10000";
|
||||
|
||||
chunkName ="chunk";
|
||||
dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
|
||||
|
||||
|
||||
}
|
||||
String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,dbDir);
|
||||
log.info(parameters);
|
||||
|
||||
|
||||
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
|
||||
cmd = String.format(cmd, dbDir,"dumps.rdb",Integer.valueOf(port));
|
||||
loadRedis(cmd,serverWait);
|
||||
|
||||
String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
|
||||
cmd = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner));
|
||||
loadRedis(cmd,serverWait);
|
||||
|
||||
JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
|
||||
JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000);
|
||||
|
||||
|
||||
ScanResult<String> scan;
|
||||
try (Jedis outer = outerPool.getResource()) {
|
||||
while (outer.ping()== "PONG"){
|
||||
log.info("wait");
|
||||
}
|
||||
|
||||
ScanParams sc = new ScanParams();
|
||||
//150000000
|
||||
sc.count(150000000);
|
||||
sc.match("*");
|
||||
|
||||
scan = outer.scan("0", sc);
|
||||
int size = scan.getResult().size();
|
||||
log.info("Scanning " + String.valueOf(size));
|
||||
}
|
||||
List<String> result = scan.getResult();
|
||||
|
||||
int fileCounter = 0;
|
||||
int pairCounter = 0;
|
||||
for (int i = 0; i < result.size(); i++) {
|
||||
for (int j = i + 1; j < result.size(); j++) {
|
||||
Jedis jedis = null;
|
||||
String key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j);
|
||||
try {
|
||||
jedis = innerPool.getResource();
|
||||
key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j);
|
||||
// String value = treesFileNames.get(i).split("GumTreeOutput2")[1] +","+treesFileNames.get(j).split("GumTreeOutput2")[1];
|
||||
// jedis.set(key,value);
|
||||
|
||||
jedis.hset(key, "0", result.get(i));
|
||||
jedis.hset(key, "1", result.get(j));
|
||||
pairCounter ++;
|
||||
|
||||
//10000000
|
||||
if (pairCounter % 10000000 == 0) {
|
||||
|
||||
File dbPath = new File(dbDir + "/" + chunkName);
|
||||
File savePath = new File(dbDir + "/" + "chunk" + String.valueOf(fileCounter) + ".rdb");
|
||||
try {
|
||||
jedis.save();
|
||||
log.info("saving key {} chunk {}",key,fileCounter);
|
||||
while (jedis.ping() == "PONG") {
|
||||
log.info("wait");
|
||||
}
|
||||
|
||||
|
||||
Files.copy(dbPath.toPath(), savePath.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
fileCounter++;
|
||||
jedis.flushDB();
|
||||
while (jedis.ping() == "PONG") {
|
||||
log.info("wait");
|
||||
}
|
||||
|
||||
} catch (IOException e) {
|
||||
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}catch (Exception e) {
|
||||
log.error(e.toString() + " {}", (key));
|
||||
|
||||
|
||||
}finally {
|
||||
if (jedis != null) {
|
||||
jedis.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// comparePairs(inputPath, innerPool,outerPool, serverWait,chunkName,dbDir,numOfWorkers);
|
||||
|
||||
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
|
||||
stopServer = String.format(stopServer,Integer.valueOf(portInner));
|
||||
loadRedis(stopServer,serverWait);
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
+2
-4
@@ -1,4 +1,4 @@
|
||||
package edu.lu.uni.serval.FixPatternParser.violations;
|
||||
package edu.lu.uni.serval.FixPatternParser.cluster;
|
||||
|
||||
import com.github.gumtreediff.actions.ActionGenerator;
|
||||
import com.github.gumtreediff.actions.model.Action;
|
||||
@@ -9,13 +9,11 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.Jedis;
|
||||
import redis.clients.jedis.JedisPool;
|
||||
import redis.clients.jedis.JedisPoolConfig;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader.getSimpliedTree;
|
||||
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.getSimpliedTree;
|
||||
|
||||
/**
|
||||
* Created by anilkoyuncu on 03/04/2018.
|
||||
+6
-88
@@ -1,10 +1,5 @@
|
||||
package edu.lu.uni.serval.FixPatternParser.violations;
|
||||
package edu.lu.uni.serval.FixPatternParser.cluster;
|
||||
|
||||
import com.github.gumtreediff.actions.ActionGenerator;
|
||||
import com.github.gumtreediff.actions.model.Action;
|
||||
import com.github.gumtreediff.matchers.Matcher;
|
||||
import com.github.gumtreediff.matchers.Matchers;
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -14,14 +9,14 @@ import redis.clients.jedis.JedisPoolConfig;
|
||||
import redis.clients.jedis.ScanResult;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader.loadRedis;
|
||||
import static edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader.getSimpliedTree;
|
||||
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis;
|
||||
|
||||
/**
|
||||
* Created by anilkoyuncu on 03/04/2018.
|
||||
@@ -156,82 +151,5 @@ public class StoreFile {
|
||||
return Base64.getEncoder().encodeToString(baos.toByteArray());
|
||||
}
|
||||
|
||||
public void storeBinary(String name , String inputPath, String innerPort) {
|
||||
JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000);
|
||||
Map<String, String> resultMap;
|
||||
try (Jedis jedis = pool.getResource()) {
|
||||
resultMap = jedis.hgetAll(name);
|
||||
}
|
||||
String[] split = name.split("_");
|
||||
|
||||
|
||||
String i = split[1];
|
||||
String j = split[2];
|
||||
String firstValue = resultMap.get("0");
|
||||
String secondValue = resultMap.get("1");
|
||||
|
||||
String[] firstValueSplit = firstValue.split("GumTreeOutput2");
|
||||
String[] secondValueSplit = secondValue.split("GumTreeOutput2");
|
||||
|
||||
|
||||
|
||||
if (firstValueSplit.length == 1) {
|
||||
firstValue = inputPath + firstValueSplit[0];
|
||||
} else {
|
||||
firstValue = inputPath + firstValueSplit[1];
|
||||
}
|
||||
|
||||
if (secondValueSplit.length == 1) {
|
||||
secondValue = inputPath + secondValueSplit[0];
|
||||
} else {
|
||||
secondValue = inputPath + secondValueSplit[1];
|
||||
}
|
||||
|
||||
try {
|
||||
ITree oldTree = getSimpliedTree(firstValue);
|
||||
|
||||
ITree newTree = getSimpliedTree(secondValue);
|
||||
|
||||
Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
|
||||
m.match();
|
||||
|
||||
|
||||
ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings());
|
||||
ag.generate();
|
||||
List<Action> actions = ag.getActions();
|
||||
|
||||
double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree);
|
||||
String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1);
|
||||
double diceSimilarity1 = m.diceSimilarity(oldTree, newTree);
|
||||
String diceSimilarity = String.format("%1.2f", diceSimilarity1);
|
||||
double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree);
|
||||
String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1);
|
||||
|
||||
String editDistance = String.valueOf(actions.size());
|
||||
|
||||
String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
|
||||
|
||||
|
||||
if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
|
||||
|| ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) {
|
||||
String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j);
|
||||
log.info(matchKey);
|
||||
try (Jedis jedis = pool.getResource()) {
|
||||
jedis.select(1);
|
||||
jedis.set(matchKey, result);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
try (Jedis jedis = pool.getResource()) {
|
||||
jedis.del("pair_" + (String.valueOf(i)) + "_" + String.valueOf(j));
|
||||
}
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error(e.toString() + " {}", (name));
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,767 @@
|
||||
package edu.lu.uni.serval.FixPatternParser.cluster;
|
||||
|
||||
import com.github.gumtreediff.actions.ActionGenerator;
|
||||
import com.github.gumtreediff.actions.model.*;
|
||||
import com.github.gumtreediff.matchers.Matcher;
|
||||
import com.github.gumtreediff.matchers.Matchers;
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
import edu.lu.uni.serval.gumtree.GumTreeComparer;
|
||||
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
|
||||
import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper;
|
||||
import edu.lu.uni.serval.utils.FileHelper;
|
||||
import edu.lu.uni.serval.utils.ListSorter;
|
||||
import org.javatuples.Pair;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.*;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.time.Duration;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis;
|
||||
|
||||
/**
|
||||
* Created by anilkoyuncu on 19/03/2018.
|
||||
*/
|
||||
public class TreeLoaderClusterL1 {
|
||||
|
||||
private static int resultType;
|
||||
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(TreeLoaderClusterL1.class);
|
||||
|
||||
public static void main(String[] args){
|
||||
|
||||
String inputPath;
|
||||
String outputPath;
|
||||
String port;
|
||||
String portInner;
|
||||
String serverWait;
|
||||
if (args.length > 0) {
|
||||
inputPath = args[0];
|
||||
outputPath = args[1];
|
||||
port = args[2];
|
||||
serverWait = args[3];
|
||||
portInner = args[4];
|
||||
|
||||
} else {
|
||||
// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2/";
|
||||
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/redis";
|
||||
outputPath = "/Users/anilkoyuncu/bugStudy/dataset/";
|
||||
port = "6379";
|
||||
portInner = "6380";
|
||||
serverWait = "10000";
|
||||
|
||||
|
||||
}
|
||||
|
||||
String cmd = "bash "+inputPath + "/" + "startServer.sh" +" %s %s %s";
|
||||
cmd = String.format(cmd, inputPath,"level1.rdb",Integer.valueOf(port));
|
||||
loadRedis(cmd,serverWait);
|
||||
|
||||
JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
|
||||
|
||||
|
||||
String level1Path = inputPath + "/level1";
|
||||
File chunks = new File(level1Path);
|
||||
File[] listFiles = chunks.listFiles();
|
||||
Stream<File> stream = Arrays.stream(listFiles);
|
||||
List<File> dbs = stream
|
||||
.filter(x -> x.getName().endsWith(".rdb"))
|
||||
.collect(Collectors.toList());
|
||||
for (File db : dbs) {
|
||||
String cmdInner = "bash "+level1Path + "/" + "startServer.sh" +" %s %s %s";
|
||||
cmdInner = String.format(cmdInner, level1Path,db.getName(),Integer.valueOf(portInner));
|
||||
loadRedis(cmdInner,serverWait);
|
||||
JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000);
|
||||
|
||||
Jedis inner = null;
|
||||
|
||||
try {
|
||||
inner = innerPool.getResource();
|
||||
|
||||
inner.select(1);
|
||||
ScanParams sc = new ScanParams();
|
||||
//150000000
|
||||
sc.count(150000000);
|
||||
sc.match("match_*");
|
||||
|
||||
ScanResult<String> scan; scan = inner.scan("0", sc);
|
||||
int size = scan.getResult().size();
|
||||
log.info("Scanning " + String.valueOf(size));
|
||||
|
||||
List<String> result = scan.getResult();
|
||||
|
||||
for (String key : result) {
|
||||
String value = inner.get(key);
|
||||
Jedis outer = null;
|
||||
try {
|
||||
outer = outerPool.getResource();
|
||||
outer.set(key,value);
|
||||
} catch (Exception e) {
|
||||
log.error(e.toString() + " {}", (key));
|
||||
|
||||
|
||||
}finally {
|
||||
if (outer != null) {
|
||||
outer.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error(e.toString() + " {}", (db.getName()));
|
||||
|
||||
|
||||
}finally {
|
||||
if (inner != null) {
|
||||
inner.close();
|
||||
}
|
||||
innerPool.close();
|
||||
}
|
||||
|
||||
String stopServer = "bash "+level1Path + "/" + "stopServer.sh" +" %s";
|
||||
stopServer = String.format(stopServer,Integer.valueOf(portInner));
|
||||
loadRedis(stopServer,serverWait);
|
||||
|
||||
}
|
||||
|
||||
|
||||
// calculatePairsOfClusters(inputPath, outputPath);
|
||||
// mainCompare(inputPath,port,pairsCSVPath,importScript);
|
||||
// calculatePairs(inputPath, outputPath);
|
||||
// processMessages(inputPath,outputPath);
|
||||
// evaluateResults(inputPath,outputPath);
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static void mainCompare(String inputPath,String port,String pairsCSVPath,String importScript) {
|
||||
|
||||
String cmd;
|
||||
cmd = "bash " + importScript +" %s";
|
||||
|
||||
JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
|
||||
|
||||
|
||||
File folder = new File(pairsCSVPath);
|
||||
File[] listOfFiles = folder.listFiles();
|
||||
Stream<File> stream = Arrays.stream(listOfFiles);
|
||||
List<File> folders = stream
|
||||
.filter(x -> !x.getName().startsWith("."))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
for (File f:folders){
|
||||
|
||||
if(f.getName().startsWith("cluster76")) {
|
||||
|
||||
|
||||
try (Jedis jedis = jedisPool.getResource()) {
|
||||
// do operations with jedis resource
|
||||
ScanParams sc = new ScanParams();
|
||||
sc.count(150000000);
|
||||
sc.match("pair_[0-9]*");
|
||||
|
||||
log.info("Scanning");
|
||||
ScanResult<String> scan = jedis.scan("0", sc);
|
||||
int size = scan.getResult().size();
|
||||
|
||||
if (size == 0) {
|
||||
// loadRedis(cmd, f);
|
||||
|
||||
scan = jedis.scan("0", sc);
|
||||
size = scan.getResult().size();
|
||||
|
||||
}
|
||||
log.info("Scanned " + String.valueOf(size));
|
||||
|
||||
|
||||
String clusterName = f.getName().replaceAll("[^0-9]+", "");
|
||||
|
||||
|
||||
//76
|
||||
|
||||
scan.getResult().parallelStream()
|
||||
.forEach(m -> coreCompare(m, inputPath, jedisPool, clusterName));
|
||||
|
||||
|
||||
jedis.save();
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public static Pair<ITree,String> getTree(String firstValue){
|
||||
String gumTreeInput = "/Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/";
|
||||
String[] split2 = firstValue.split("/");
|
||||
String cluster = split2[1];
|
||||
|
||||
|
||||
File folder = new File("/Users/anilkoyuncu/bugStudy/code/python/cluster/"+cluster);
|
||||
File[] listOfFiles = folder.listFiles();
|
||||
Stream<File> stream = Arrays.stream(listOfFiles);
|
||||
List<File> folders = stream
|
||||
.filter(x -> !x.getName().startsWith(".") && x.getName().startsWith(split2[2]))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
|
||||
|
||||
String[] split1 = folders.get(0).getName().split(".txt_");
|
||||
String s = split1[0];
|
||||
String[] splitPJ = split1[1].split("_");
|
||||
String project = splitPJ[1];
|
||||
String actionSetPosition = splitPJ[0];
|
||||
|
||||
File prevFile = new File(gumTreeInput + project+ "/" + "prevFiles/prev_" + s + ".java");// previous file
|
||||
File revFile = new File(gumTreeInput + project+ "/" + "revFiles/" + s + ".java");//rev file
|
||||
|
||||
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile);
|
||||
|
||||
HierarchicalActionSet actionSet = actionSets.get(Integer.valueOf(actionSetPosition));
|
||||
// for (HierarchicalActionSet actionSet : actionSets) {
|
||||
|
||||
|
||||
|
||||
ITree actionTree= null;
|
||||
ITree test2 = null;
|
||||
getActionTree(actionSet);
|
||||
ITree node = actionSet.getNode();
|
||||
List<ITree> descendants = node.getDescendants();
|
||||
for (ITree descendant : descendants) {
|
||||
if(descendant.getType() <= 100){
|
||||
descendant.setType(104);
|
||||
}
|
||||
}
|
||||
node.setParent(null);
|
||||
|
||||
|
||||
// }
|
||||
// }
|
||||
|
||||
Pair<ITree, String> pair = new Pair<>(node,project);
|
||||
return pair;
|
||||
}
|
||||
|
||||
|
||||
public static void getActionTree(HierarchicalActionSet actionSet){
|
||||
|
||||
|
||||
int newType = 0;
|
||||
|
||||
Action action = actionSet.getAction();
|
||||
if (action instanceof Update){
|
||||
newType = 101;
|
||||
}else if(action instanceof Insert){
|
||||
newType =100;
|
||||
}else if(action instanceof Move){
|
||||
newType = 102;
|
||||
}else if(action instanceof Delete){
|
||||
newType=103;
|
||||
}else{
|
||||
new Exception("unknow action");
|
||||
}
|
||||
actionSet.getNode().setType(newType);
|
||||
// actionSet.getNode().setLabel("");
|
||||
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
|
||||
if (subActions.size() != 0){
|
||||
for (HierarchicalActionSet subAction : subActions) {
|
||||
getActionTree(subAction);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
// public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children){
|
||||
//
|
||||
// int newType = 0;
|
||||
//
|
||||
// Action action = actionSet.getAction();
|
||||
// if (action instanceof Update){
|
||||
// newType = 101;
|
||||
// }else if(action instanceof Insert){
|
||||
// newType =100;
|
||||
// }else if(action instanceof Move){
|
||||
// newType = 102;
|
||||
// }else if(action instanceof Delete){
|
||||
// newType=103;
|
||||
// }else{
|
||||
// new Exception("unknow action");
|
||||
// }
|
||||
// if(actionSet.getParent() == null){
|
||||
// //root
|
||||
//
|
||||
// parent = new Tree(newType,"");
|
||||
// }else{
|
||||
// children = new Tree(newType,"");
|
||||
// parent.addChild(children);
|
||||
// }
|
||||
// List<HierarchicalActionSet> subActions = actionSet.getSubActions();
|
||||
// if (subActions.size() != 0){
|
||||
// for (HierarchicalActionSet subAction : subActions) {
|
||||
//
|
||||
// if(actionSet.getParent() == null){
|
||||
// children = parent;
|
||||
// }
|
||||
// getActionTree(subAction,children,null);
|
||||
//
|
||||
// }
|
||||
//
|
||||
//
|
||||
// }
|
||||
// return parent;
|
||||
// }
|
||||
|
||||
|
||||
|
||||
private static void coreCompare(String name , String inputPath, JedisPool jedisPool,String clusterName) {
|
||||
|
||||
|
||||
try (Jedis jedis = jedisPool.getResource()) {
|
||||
|
||||
|
||||
Map<String, String> resultMap = jedis.hgetAll(name);
|
||||
|
||||
resultMap.get("0");
|
||||
|
||||
|
||||
String[] split = name.split("_");
|
||||
String i = null;
|
||||
String j =null;
|
||||
try {
|
||||
i = split[1];
|
||||
j = split[2];
|
||||
}
|
||||
catch (Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
String firstValue = resultMap.get("0");
|
||||
String secondValue = resultMap.get("1");
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// firstValue = inputPath + firstValue;
|
||||
// secondValue = inputPath + secondValue;
|
||||
|
||||
// String[] firstValueSplit = firstValue.split("/");
|
||||
// String[] secondValueSplit = secondValue.split("/");
|
||||
//
|
||||
// if (firstValueSplit.length == 1) {
|
||||
// firstValue = inputPath + firstValueSplit[0];
|
||||
// } else {
|
||||
// firstValue = inputPath + firstValueSplit[1];
|
||||
// }
|
||||
//
|
||||
// if (secondValueSplit.length == 1) {
|
||||
// secondValue = inputPath + secondValueSplit[0];
|
||||
// } else {
|
||||
// secondValue = inputPath + secondValueSplit[1];
|
||||
// }
|
||||
|
||||
try {
|
||||
Pair<ITree, String> oldPair = getTree(firstValue);
|
||||
Pair<ITree, String> newPair = getTree(secondValue);
|
||||
|
||||
ITree oldTree = oldPair.getValue0();
|
||||
ITree newTree = newPair.getValue0();
|
||||
|
||||
String oldProject = oldPair.getValue1();
|
||||
String newProject = newPair.getValue1();
|
||||
|
||||
|
||||
|
||||
Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
|
||||
m.match();
|
||||
|
||||
|
||||
ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings());
|
||||
ag.generate();
|
||||
List<Action> actions = ag.getActions();
|
||||
|
||||
String resultKey = "result_" + (String.valueOf(i)) + "_" + String.valueOf(j);
|
||||
double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree);
|
||||
String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1);
|
||||
double diceSimilarity1 = m.diceSimilarity(oldTree, newTree);
|
||||
String diceSimilarity = String.format("%1.2f", diceSimilarity1);
|
||||
double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree);
|
||||
String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1);
|
||||
|
||||
String editDistance = String.valueOf(actions.size());
|
||||
// jedis.select(1);
|
||||
String result = resultMap.get("0") + "," + oldProject +"," + resultMap.get("1") + "," +newProject+ "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
|
||||
// jedis.set(resultKey, result);
|
||||
|
||||
if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
|
||||
|| ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) {
|
||||
String matchKey = "match-"+clusterName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j);
|
||||
jedis.select(1);
|
||||
jedis.set(matchKey, result);
|
||||
}
|
||||
jedis.select(0);
|
||||
String pairKey = "pair_" + (String.valueOf(i)) + "_" + String.valueOf(j);
|
||||
jedis.del(pairKey);
|
||||
|
||||
// log.info("Completed " + resultKey);
|
||||
|
||||
}catch (Exception e){
|
||||
log.error(e.toString() + " {}",(name));
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
protected static List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree2(File prevFile, File revFile) {
|
||||
List<HierarchicalActionSet> actionSets = new ArrayList<>();
|
||||
// GumTree results
|
||||
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
|
||||
if (gumTreeResults == null) {
|
||||
resultType = 1;
|
||||
return null;
|
||||
} else if (gumTreeResults.size() == 0){
|
||||
resultType = 2;
|
||||
return actionSets;
|
||||
} else {
|
||||
// Regroup GumTre results.
|
||||
List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
|
||||
// for (HierarchicalActionSet actionSet : allActionSets) {
|
||||
// String astNodeType = actionSet.getAstNodeType();
|
||||
// if (astNodeType.endsWith("Statement") || "FieldDeclaration".equals(astNodeType)) {
|
||||
// actionSets.add(actionSet);
|
||||
// }
|
||||
// }
|
||||
|
||||
// Filter out modified actions of changing method names, method parameters, variable names and field names in declaration part.
|
||||
// variable effects range, sub-actions are these kinds of modification?
|
||||
// actionSets.addAll(new ActionFilter().filterOutUselessActions(allActionSets));
|
||||
|
||||
ListSorter<HierarchicalActionSet> sorter = new ListSorter<>(allActionSets);
|
||||
actionSets = sorter.sortAscending();
|
||||
|
||||
if (actionSets.size() == 0) {
|
||||
resultType = 3;
|
||||
}
|
||||
|
||||
return actionSets;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
orginal calculate pairs, from all dumps of the projects
|
||||
*/
|
||||
public static void calculatePairs(String inputPath, String outputPath) {
|
||||
File folder = new File(inputPath);
|
||||
File[] listOfFiles = folder.listFiles();
|
||||
Stream<File> stream = Arrays.stream(listOfFiles);
|
||||
List<File> pjs = stream
|
||||
.filter(x -> !x.getName().startsWith("."))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
List<File> fileToCompare = new ArrayList<>();
|
||||
for (File pj : pjs) {
|
||||
File[] files = pj.listFiles(new FilenameFilter() {
|
||||
public boolean accept(File dir, String name) {
|
||||
return name.startsWith("ASTDumps");
|
||||
}
|
||||
});
|
||||
Collections.addAll(fileToCompare, files[0].listFiles());
|
||||
|
||||
}
|
||||
System.out.println("a");
|
||||
// compareAll(fileToCompare);
|
||||
readMessageFiles(fileToCompare, outputPath);
|
||||
}
|
||||
|
||||
/*
|
||||
pairs of each cluster
|
||||
*/
|
||||
public static void calculatePairsOfClusters(String inputPath, String outputPath) {
|
||||
File folder = new File(inputPath);
|
||||
File[] listOfFiles = folder.listFiles();
|
||||
Stream<File> stream = Arrays.stream(listOfFiles);
|
||||
List<File> pjs = stream
|
||||
.filter(x -> !x.getName().startsWith("."))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
FileHelper.createDirectory(outputPath + "pairs/");
|
||||
for (File pj : pjs) {
|
||||
File[] files = pj.listFiles();
|
||||
List<File> fileList = Arrays.asList(files);
|
||||
|
||||
readMessageFilesCluster(fileList, outputPath,inputPath,pj.getName());
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static void processMessages(String inputPath, String outputPath) {
|
||||
File folder = new File(outputPath + "pairs_splitted/");
|
||||
File[] listOfFiles = folder.listFiles();
|
||||
Stream<File> stream = Arrays.stream(listOfFiles);
|
||||
List<File> pjs = stream
|
||||
.filter(x -> !x.getName().startsWith("."))
|
||||
.collect(Collectors.toList());
|
||||
FileHelper.createDirectory(outputPath + "comparison_splitted/");
|
||||
pjs.parallelStream()
|
||||
.forEach(m -> coreLoop(m, outputPath,inputPath));
|
||||
}
|
||||
|
||||
private static void readMessageFilesCluster(List<File> folders, String outputPath,String inputPath,String cluster) {
|
||||
|
||||
List<String> treesFileNames = new ArrayList<>();
|
||||
|
||||
|
||||
for (File target : folders) {
|
||||
|
||||
treesFileNames.add(target.toString());
|
||||
}
|
||||
|
||||
log.info("Calculating pairs");
|
||||
// treesFileNames = treesFileNames.subList(0,100);
|
||||
|
||||
String filename = "cluster" + cluster;
|
||||
byte [] buf = new byte[0];
|
||||
String line = null;
|
||||
try {
|
||||
|
||||
FileOutputStream fos = new FileOutputStream(outputPath + "pairs/" +filename+".txt");
|
||||
DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos));
|
||||
|
||||
|
||||
|
||||
for (int i = 0; i < treesFileNames.size(); i++) {
|
||||
for (int j = i + 1; j < treesFileNames.size(); j++) {
|
||||
|
||||
|
||||
|
||||
line = String.valueOf(i) +"\t" + String.valueOf(j) + "\t" + treesFileNames.get(i).replace(inputPath,"") + "\t" + treesFileNames.get(j).replace(inputPath,"")+"\n";
|
||||
outStream.write(line.getBytes());
|
||||
|
||||
}
|
||||
}
|
||||
outStream.close();
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}catch (java.nio.BufferOverflowException e) {
|
||||
log.error(line);
|
||||
log.error(String.valueOf(buf.length));
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
log.info("Done pairs");
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static ITree getSimpliedTree(String fn) {
|
||||
ITree tree = null;
|
||||
try {
|
||||
FileInputStream fi = new FileInputStream(new File(fn));
|
||||
ObjectInputStream oi = new ObjectInputStream(fi);
|
||||
tree = (ITree) oi.readObject();
|
||||
oi.close();
|
||||
fi.close();
|
||||
|
||||
|
||||
} catch (FileNotFoundException e) {
|
||||
log.error("File not found");
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
log.error("Error initializing stream");
|
||||
e.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
// tree.setLabel("");
|
||||
tree.setParent(null);
|
||||
// List<ITree> descendants = tree.getDescendants();
|
||||
// for (ITree descendant : descendants) {
|
||||
// descendant.setLabel("");
|
||||
// }
|
||||
|
||||
return tree;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static void coreLoop(File mes, String outputPath,String inputPath) {
|
||||
try {
|
||||
|
||||
log.info("Starting in coreLoop");
|
||||
|
||||
BufferedReader br = null;
|
||||
String sCurrentLine = null;
|
||||
BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "comparison_splitted/" + "output_" + mes.getName()));
|
||||
|
||||
br = new BufferedReader(
|
||||
new FileReader(mes));
|
||||
while ((sCurrentLine = br.readLine()) != null) {
|
||||
String currentLine = sCurrentLine;
|
||||
String[] split = currentLine.split("\t");
|
||||
String i = split[0];
|
||||
String j = split[1];
|
||||
String firstValue = split[2];
|
||||
String secondValue = split[3];
|
||||
|
||||
firstValue = inputPath + firstValue.split("GumTreeOutput2")[1];
|
||||
secondValue = inputPath + secondValue.split("GumTreeOutput2")[1];
|
||||
|
||||
ITree oldTree = getSimpliedTree(firstValue);
|
||||
|
||||
ITree newTree = getSimpliedTree(secondValue);
|
||||
|
||||
Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
|
||||
m.match();
|
||||
|
||||
ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings());
|
||||
ag.generate();
|
||||
List<Action> actions = ag.getActions();
|
||||
writer.write(String.valueOf(i));
|
||||
writer.write("\t");
|
||||
writer.write(String.valueOf(j));
|
||||
writer.write("\t");
|
||||
|
||||
writer.write(String.format("%1.2f", m.chawatheSimilarity(oldTree, newTree)));
|
||||
writer.write("\t");
|
||||
writer.write(String.format("%1.2f", m.diceSimilarity(oldTree, newTree)));
|
||||
writer.write("\t");
|
||||
writer.write(String.format("%1.2f", m.jaccardSimilarity(oldTree, newTree)));
|
||||
writer.write("\t");
|
||||
writer.write(String.valueOf(actions.size()));
|
||||
writer.write("\t");
|
||||
writer.write(firstValue);
|
||||
writer.write("\t");
|
||||
writer.write(secondValue);
|
||||
writer.write("\n");
|
||||
|
||||
|
||||
}
|
||||
writer.close();
|
||||
} catch (FileNotFoundException e) {
|
||||
log.error("File not found");
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
log.error("Error initializing stream");
|
||||
e.printStackTrace();
|
||||
|
||||
}
|
||||
log.info("Completed output_" + mes.getName());
|
||||
}
|
||||
|
||||
private static void readMessageFiles(List<File> folders, String outputPath) {
|
||||
|
||||
List<String> treesFileNames = new ArrayList<>();
|
||||
|
||||
|
||||
for (File target : folders) {
|
||||
|
||||
treesFileNames.add(target.toString());
|
||||
}
|
||||
FileHelper.createDirectory(outputPath + "pairs/");
|
||||
log.info("Calculating pairs");
|
||||
// treesFileNames = treesFileNames.subList(0,100);
|
||||
byte [] buf = new byte[0];
|
||||
String line = null;
|
||||
try {
|
||||
|
||||
FileChannel rwChannel = new RandomAccessFile(outputPath + "pairs/" +"textfile.txt", "rw").getChannel();
|
||||
ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE);
|
||||
int fileCounter = 0;
|
||||
|
||||
|
||||
for (int i = 0; i < treesFileNames.size(); i++) {
|
||||
for (int j = i + 1; j < treesFileNames.size(); j++) {
|
||||
|
||||
|
||||
|
||||
line = String.valueOf(i) +"\t" + String.valueOf(j) + "\t" + treesFileNames.get(i).replace("/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2","") + "\t" + treesFileNames.get(j).replace("/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2","")+"\n";
|
||||
buf = line.getBytes();
|
||||
if(wrBuf.remaining() > 500) {
|
||||
wrBuf.put(buf);
|
||||
}else{
|
||||
log.info("Next pair dump");
|
||||
fileCounter++;
|
||||
rwChannel = new RandomAccessFile(outputPath+"pairs/" +"textfile"+String.valueOf(fileCounter)+".txt", "rw").getChannel();
|
||||
wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
rwChannel.close();
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}catch (java.nio.BufferOverflowException e) {
|
||||
log.error(line);
|
||||
log.error(String.valueOf(buf.length));
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
log.info("Done pairs");
|
||||
}
|
||||
|
||||
static final JedisPoolConfig poolConfig = buildPoolConfig();
|
||||
|
||||
|
||||
private static JedisPoolConfig buildPoolConfig() {
|
||||
final JedisPoolConfig poolConfig = new JedisPoolConfig();
|
||||
poolConfig.setMaxTotal(128);
|
||||
poolConfig.setMaxIdle(128);
|
||||
poolConfig.setMinIdle(16);
|
||||
poolConfig.setTestOnBorrow(true);
|
||||
poolConfig.setTestOnReturn(true);
|
||||
poolConfig.setTestWhileIdle(true);
|
||||
poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis());
|
||||
poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis());
|
||||
poolConfig.setNumTestsPerEvictionRun(3);
|
||||
poolConfig.setBlockWhenExhausted(true);
|
||||
|
||||
return poolConfig;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// return msgFiles;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,655 +0,0 @@
|
||||
package edu.lu.uni.serval.FixPatternParser.violations;
|
||||
|
||||
import akka.actor.ActorRef;
|
||||
import akka.actor.ActorSystem;
|
||||
import akka.actor.Props;
|
||||
import com.github.gumtreediff.actions.ActionGenerator;
|
||||
import com.github.gumtreediff.actions.model.*;
|
||||
import com.github.gumtreediff.matchers.Matcher;
|
||||
import com.github.gumtreediff.matchers.Matchers;
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
import com.github.gumtreediff.tree.Tree;
|
||||
import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap;
|
||||
import edu.lu.uni.serval.MultipleThreadsParser.*;
|
||||
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
|
||||
import edu.lu.uni.serval.utils.FileHelper;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.*;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* Created by anilkoyuncu on 19/03/2018.
|
||||
*/
|
||||
public class AkkaTreeLoader {
|
||||
|
||||
private static class StreamGobbler implements Runnable {
|
||||
private InputStream inputStream;
|
||||
private Consumer<String> consumer;
|
||||
|
||||
public StreamGobbler(InputStream inputStream, Consumer<String> consumer) {
|
||||
this.inputStream = inputStream;
|
||||
this.consumer = consumer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
new BufferedReader(new InputStreamReader(inputStream)).lines()
|
||||
.forEach(consumer);
|
||||
}
|
||||
}
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class);
|
||||
|
||||
|
||||
public static void loadRedis(String cmd,String serverWait){
|
||||
Process process;
|
||||
|
||||
try {
|
||||
// String comd = String.format(cmd, f.getAbsoluteFile());
|
||||
process = Runtime.getRuntime()
|
||||
|
||||
.exec(cmd);
|
||||
|
||||
|
||||
StreamGobbler streamGobbler =
|
||||
new StreamGobbler(process.getInputStream(), System.out::println);
|
||||
Executors.newSingleThreadExecutor().submit(streamGobbler);
|
||||
// int exitCode = process.waitFor();
|
||||
// assert exitCode == 0;
|
||||
Thread.sleep(Integer.valueOf(serverWait));
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
log.info("Load done");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
|
||||
String inputPath;
|
||||
// String outputPath;
|
||||
String port;
|
||||
String portInner;
|
||||
// String pairsCSVPath;
|
||||
String importScript;
|
||||
// String pairsCompletedPath;
|
||||
String serverWait;
|
||||
// String option;
|
||||
String dbDir;
|
||||
String chunkName;
|
||||
String numOfWorkers;
|
||||
if (args.length > 0) {
|
||||
inputPath = args[0];
|
||||
portInner = args[1];
|
||||
serverWait = args[2];
|
||||
// option = args[4];
|
||||
chunkName = args[3];
|
||||
numOfWorkers = args[4];
|
||||
dbDir = args[5];
|
||||
port = args[6];
|
||||
// pairsCSVPath = args[3];
|
||||
// importScript = args[4];
|
||||
// pairsCompletedPath = args[3];
|
||||
} else {
|
||||
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
|
||||
// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/";
|
||||
port = "6399";
|
||||
portInner = "6380";
|
||||
serverWait = "10000";
|
||||
// option = "COMP";
|
||||
// pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test";
|
||||
// importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh";
|
||||
// pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed";
|
||||
chunkName ="chunk5.rdb";
|
||||
dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
|
||||
numOfWorkers = "1";
|
||||
|
||||
}
|
||||
String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \nnumOfWorks %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,numOfWorkers,dbDir);
|
||||
log.info(parameters);
|
||||
|
||||
// if (option.equals("CALC")) {
|
||||
// calculatePairs(inputPath, port);
|
||||
// log.info("Calculate pairs done");
|
||||
// }else {
|
||||
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
|
||||
cmd = String.format(cmd, dbDir,"dumps.rdb",Integer.valueOf(port));
|
||||
loadRedis(cmd,serverWait);
|
||||
|
||||
String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
|
||||
cmd = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner));
|
||||
loadRedis(cmd,serverWait);
|
||||
|
||||
JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
|
||||
JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000);
|
||||
|
||||
|
||||
|
||||
comparePairs(inputPath, innerPool,outerPool, serverWait,chunkName,dbDir,numOfWorkers);
|
||||
|
||||
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
|
||||
stopServer = String.format(stopServer,Integer.valueOf(portInner));
|
||||
loadRedis(stopServer,serverWait);
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static void comparePairs(String inputPath, JedisPool innerPool,JedisPool outerPool,String serverWait,String chunkName, String dbDir,String numOfWorkers){
|
||||
|
||||
|
||||
ScanResult<String> scan;
|
||||
try (Jedis inner = innerPool.getResource()) {
|
||||
while (inner.ping()== "PONG"){
|
||||
log.info("wait");
|
||||
}
|
||||
|
||||
ScanParams sc = new ScanParams();
|
||||
//150000000
|
||||
sc.count(150000000);
|
||||
sc.match("pair_*");
|
||||
|
||||
scan = inner.scan("0", sc);
|
||||
int size = scan.getResult().size();
|
||||
log.info("Scanning " + String.valueOf(size));
|
||||
}
|
||||
List<String> result = scan.getResult();
|
||||
|
||||
|
||||
// ActorSystem system = null;
|
||||
// ActorRef parsingActor = null;
|
||||
// final WorkMessage msg = new WorkMessage(0, result,innerPort,inputPath,dbDir,serverWait);
|
||||
// try {
|
||||
//
|
||||
// log.info("Akka begins...");
|
||||
// system = ActorSystem.create("Tree-System");
|
||||
// parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers),dbDir,innerPort,serverWait), "tree-actor");
|
||||
// parsingActor.tell(msg, ActorRef.noSender());
|
||||
// } catch (Exception e) {
|
||||
// system.shutdown();
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// greeter.tell();
|
||||
result
|
||||
.parallelStream()
|
||||
.forEach(m ->
|
||||
{
|
||||
Compare compare = new Compare();
|
||||
compare.coreCompare(m, innerPool, outerPool);
|
||||
;
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
|
||||
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
// private static void coreCompare(String name , String inputPath, String innerPort) {
|
||||
// JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000);
|
||||
// Map<String, String> resultMap;
|
||||
// try (Jedis jedis = pool.getResource()) {
|
||||
// resultMap = jedis.hgetAll(name);
|
||||
// }
|
||||
// String[] split = name.split("_");
|
||||
//
|
||||
//
|
||||
// String i = split[1];
|
||||
// String j = split[2];
|
||||
// String firstValue = resultMap.get("0");
|
||||
// String secondValue = resultMap.get("1");
|
||||
//
|
||||
// String[] firstValueSplit = firstValue.split("GumTreeOutput2");
|
||||
// String[] secondValueSplit = secondValue.split("GumTreeOutput2");
|
||||
//
|
||||
// if (firstValueSplit.length == 1) {
|
||||
// firstValue = inputPath + firstValueSplit[0];
|
||||
// } else {
|
||||
// firstValue = inputPath + firstValueSplit[1];
|
||||
// }
|
||||
//
|
||||
// if (secondValueSplit.length == 1) {
|
||||
// secondValue = inputPath + secondValueSplit[0];
|
||||
// } else {
|
||||
// secondValue = inputPath + secondValueSplit[1];
|
||||
// }
|
||||
//
|
||||
// try {
|
||||
// ITree oldTree = getSimpliedTree(firstValue);
|
||||
//
|
||||
// ITree newTree = getSimpliedTree(secondValue);
|
||||
//
|
||||
// Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
|
||||
// m.match();
|
||||
//
|
||||
//
|
||||
// ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings());
|
||||
// ag.generate();
|
||||
// List<Action> actions = ag.getActions();
|
||||
//
|
||||
// double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree);
|
||||
// String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1);
|
||||
// double diceSimilarity1 = m.diceSimilarity(oldTree, newTree);
|
||||
// String diceSimilarity = String.format("%1.2f", diceSimilarity1);
|
||||
// double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree);
|
||||
// String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1);
|
||||
//
|
||||
// String editDistance = String.valueOf(actions.size());
|
||||
//
|
||||
// String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
|
||||
//
|
||||
//
|
||||
// if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
|
||||
// || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) {
|
||||
// String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j);
|
||||
//
|
||||
// try (Jedis jedis = pool.getResource()) {
|
||||
// jedis.select(1);
|
||||
// jedis.set(matchKey, result);
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
// try (Jedis jedis = pool.getResource()) {
|
||||
// jedis.del("pair_"+ (String.valueOf(i)) + "_" + String.valueOf(j));
|
||||
// }
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
// }catch (Exception e){
|
||||
// log.error(e.toString() + " {}",(name));
|
||||
//
|
||||
//
|
||||
// }
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
|
||||
public static void calculatePairs(String inputPath,String port) {
|
||||
File folder = new File(inputPath);
|
||||
File[] listOfFiles = folder.listFiles();
|
||||
Stream<File> stream = Arrays.stream(listOfFiles);
|
||||
List<File> pjs = stream
|
||||
.filter(x -> !x.getName().startsWith("."))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
List<File> fileToCompare = new ArrayList<>();
|
||||
for (File pj : pjs) {
|
||||
File[] files = pj.listFiles(new FilenameFilter() {
|
||||
public boolean accept(File dir, String name) {
|
||||
return name.startsWith("ActionSetDumps");
|
||||
}
|
||||
});
|
||||
Collections.addAll(fileToCompare, files[0].listFiles());
|
||||
|
||||
}
|
||||
System.out.println("a");
|
||||
// compareAll(fileToCompare);
|
||||
readMessageFiles(fileToCompare,port);
|
||||
}
|
||||
|
||||
// public static void processMessages(String inputPath, String outputPath) {
|
||||
// File folder = new File(outputPath + "pairs_splitted/");
|
||||
// File[] listOfFiles = folder.listFiles();
|
||||
// Stream<File> stream = Arrays.stream(listOfFiles);
|
||||
// List<File> pjs = stream
|
||||
// .filter(x -> !x.getName().startsWith("."))
|
||||
// .collect(Collectors.toList());
|
||||
// FileHelper.createDirectory(outputPath + "comparison_splitted/");
|
||||
// pjs.parallelStream()
|
||||
// .forEach(m -> coreLoop(m, outputPath,inputPath));
|
||||
// }
|
||||
|
||||
/** Read the object from Base64 string. */
|
||||
private static Object fromString( String s ) throws IOException ,
|
||||
ClassNotFoundException {
|
||||
byte [] data = Base64.getDecoder().decode( s );
|
||||
ObjectInputStream ois = new ObjectInputStream(
|
||||
new ByteArrayInputStream( data ) );
|
||||
Object o = ois.readObject();
|
||||
ois.close();
|
||||
return o;
|
||||
}
|
||||
public static ITree getSimpliedTree(String fn,JedisPool outerPool) {
|
||||
|
||||
ITree tree = null;
|
||||
Jedis inner = null;
|
||||
try {
|
||||
inner = outerPool.getResource();
|
||||
String s = inner.get(fn.substring(1));
|
||||
HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s);
|
||||
|
||||
ITree parent = null;
|
||||
ITree children =null;
|
||||
tree = getASTTree(actionSet, parent, children);
|
||||
tree.setParent(null);
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}finally {
|
||||
if (inner != null) {
|
||||
inner.close();
|
||||
}
|
||||
}
|
||||
return tree;
|
||||
|
||||
}
|
||||
|
||||
public static <T, E> List<T> getKeysByValue(Map<T, E> map, E value) {
|
||||
return map.entrySet()
|
||||
.stream()
|
||||
.filter(entry -> Objects.equals(entry.getValue(), value))
|
||||
.map(Map.Entry::getKey)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children){
|
||||
|
||||
int newType = 0;
|
||||
|
||||
String astNodeType = actionSet.getAstNodeType();
|
||||
List<Integer> keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType);
|
||||
|
||||
if(keysByValue.size() != 1){
|
||||
log.error("Birden cok astnodemapmapping");
|
||||
}
|
||||
newType = keysByValue.get(0);
|
||||
if(actionSet.getParent() == null){
|
||||
//root
|
||||
|
||||
parent = new Tree(newType,"");
|
||||
}else{
|
||||
children = new Tree(newType,"");
|
||||
parent.addChild(children);
|
||||
}
|
||||
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
|
||||
if (subActions.size() != 0){
|
||||
for (HierarchicalActionSet subAction : subActions) {
|
||||
|
||||
if(actionSet.getParent() == null){
|
||||
children = parent;
|
||||
}
|
||||
getASTTree(subAction,children,null);
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
return parent;
|
||||
}
|
||||
|
||||
public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children){
|
||||
|
||||
int newType = 0;
|
||||
|
||||
Action action = actionSet.getAction();
|
||||
if (action instanceof Update){
|
||||
newType = 101;
|
||||
}else if(action instanceof Insert){
|
||||
newType =100;
|
||||
}else if(action instanceof Move){
|
||||
newType = 102;
|
||||
}else if(action instanceof Delete){
|
||||
newType=103;
|
||||
}else{
|
||||
new Exception("unknow action");
|
||||
}
|
||||
if(actionSet.getParent() == null){
|
||||
//root
|
||||
|
||||
parent = new Tree(newType,"");
|
||||
}else{
|
||||
children = new Tree(newType,"");
|
||||
parent.addChild(children);
|
||||
}
|
||||
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
|
||||
if (subActions.size() != 0){
|
||||
for (HierarchicalActionSet subAction : subActions) {
|
||||
|
||||
if(actionSet.getParent() == null){
|
||||
children = parent;
|
||||
}
|
||||
getActionTree(subAction,children,null);
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
return parent;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// private static void coreLoop(File mes, String outputPath,String inputPath) {
|
||||
// try {
|
||||
//
|
||||
// log.info("Starting in coreLoop");
|
||||
//
|
||||
// BufferedReader br = null;
|
||||
// String sCurrentLine = null;
|
||||
// BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "comparison_splitted/" + "output_" + mes.getName()));
|
||||
//
|
||||
// br = new BufferedReader(
|
||||
// new FileReader(mes));
|
||||
// while ((sCurrentLine = br.readLine()) != null) {
|
||||
// String currentLine = sCurrentLine;
|
||||
// String[] split = currentLine.split("\t");
|
||||
// String i = split[0];
|
||||
// String j = split[1];
|
||||
// String firstValue = split[2];
|
||||
// String secondValue = split[3];
|
||||
//
|
||||
// firstValue = inputPath + firstValue.split("GumTreeOutput2")[1];
|
||||
// secondValue = inputPath + secondValue.split("GumTreeOutput2")[1];
|
||||
//
|
||||
// ITree oldTree = getSimpliedTree(firstValue);
|
||||
//
|
||||
// ITree newTree = getSimpliedTree(secondValue);
|
||||
//
|
||||
// Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
|
||||
// m.match();
|
||||
//
|
||||
// ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings());
|
||||
// ag.generate();
|
||||
// List<Action> actions = ag.getActions();
|
||||
// writer.write(String.valueOf(i));
|
||||
// writer.write("\t");
|
||||
// writer.write(String.valueOf(j));
|
||||
// writer.write("\t");
|
||||
//
|
||||
// writer.write(String.format("%1.2f", m.chawatheSimilarity(oldTree, newTree)));
|
||||
// writer.write("\t");
|
||||
// writer.write(String.format("%1.2f", m.diceSimilarity(oldTree, newTree)));
|
||||
// writer.write("\t");
|
||||
// writer.write(String.format("%1.2f", m.jaccardSimilarity(oldTree, newTree)));
|
||||
// writer.write("\t");
|
||||
// writer.write(String.valueOf(actions.size()));
|
||||
// writer.write("\t");
|
||||
// writer.write(firstValue);
|
||||
// writer.write("\t");
|
||||
// writer.write(secondValue);
|
||||
// writer.write("\n");
|
||||
//
|
||||
//
|
||||
// }
|
||||
// writer.close();
|
||||
// } catch (FileNotFoundException e) {
|
||||
// log.error("File not found");
|
||||
// e.printStackTrace();
|
||||
// } catch (IOException e) {
|
||||
// log.error("Error initializing stream");
|
||||
// e.printStackTrace();
|
||||
//
|
||||
// }
|
||||
// log.info("Completed output_" + mes.getName());
|
||||
// }
|
||||
|
||||
private static void readMessageFiles(List<File> folders,String port) {
|
||||
|
||||
List<String> treesFileNames = new ArrayList<>();
|
||||
|
||||
|
||||
for (File target : folders) {
|
||||
|
||||
treesFileNames.add(target.toString());
|
||||
}
|
||||
// FileHelper.createDirectory(outputPath + "pairs/");
|
||||
log.info("Calculating pairs");
|
||||
// treesFileNames = treesFileNames.subList(0,100);
|
||||
byte [] buf = new byte[0];
|
||||
String line = null;
|
||||
|
||||
|
||||
// FileChannel rwChannel = new RandomAccessFile(outputPath + "pairs/" +"textfile.txt", "rw").getChannel();
|
||||
// ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE);
|
||||
int fileCounter = 0;
|
||||
|
||||
JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
|
||||
try (Jedis jedis = jedisPool.getResource()) {
|
||||
List<String> dir = null;
|
||||
List<String> path = null;
|
||||
for (int i = 0; i < treesFileNames.size(); i++) {
|
||||
for (int j = i + 1; j < treesFileNames.size(); j++) {
|
||||
|
||||
|
||||
// do operations with jedis resource
|
||||
|
||||
String key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j);
|
||||
// String value = treesFileNames.get(i).split("GumTreeOutput2")[1] +","+treesFileNames.get(j).split("GumTreeOutput2")[1];
|
||||
// jedis.set(key,value);
|
||||
|
||||
jedis.hset(key,"0",treesFileNames.get(i).split("GumTreeOutput2")[1]);
|
||||
jedis.hset(key,"1",treesFileNames.get(j).split("GumTreeOutput2")[1]);
|
||||
//10000000
|
||||
if(Integer.compare(jedis.dbSize().intValue(),10000000) == 0){
|
||||
dir = jedis.configGet("dir");
|
||||
path = jedis.configGet("dbfilename");
|
||||
File dbPath = new File(dir.get(1)+"/"+path.get(1));
|
||||
File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb");
|
||||
try {
|
||||
jedis.save();
|
||||
while (jedis.ping()== "PONG"){
|
||||
log.info("wait");
|
||||
}
|
||||
|
||||
|
||||
|
||||
Files.copy(dbPath.toPath(),savePath.toPath());
|
||||
} catch (IOException e) {
|
||||
|
||||
e.printStackTrace();
|
||||
}
|
||||
fileCounter++;
|
||||
jedis.flushDB();
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
jedis.save();
|
||||
fileCounter++;
|
||||
File dbPath = new File(dir.get(1)+"/"+path.get(1));
|
||||
File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb");
|
||||
try {
|
||||
|
||||
while (jedis.ping()== "PONG"){
|
||||
log.info("wait");
|
||||
}
|
||||
|
||||
|
||||
|
||||
Files.copy(dbPath.toPath(),savePath.toPath());
|
||||
} catch (IOException e) {
|
||||
|
||||
e.printStackTrace();
|
||||
}
|
||||
jedis.flushDB();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
log.info("Done pairs");
|
||||
}
|
||||
|
||||
static final JedisPoolConfig poolConfig = buildPoolConfig();
|
||||
|
||||
|
||||
private static JedisPoolConfig buildPoolConfig() {
|
||||
final JedisPoolConfig poolConfig = new JedisPoolConfig();
|
||||
poolConfig.setMaxTotal(128);
|
||||
poolConfig.setMaxIdle(128);
|
||||
poolConfig.setMinIdle(16);
|
||||
poolConfig.setTestOnBorrow(false);
|
||||
poolConfig.setTestOnReturn(false);
|
||||
poolConfig.setTestWhileIdle(true);
|
||||
poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis());
|
||||
poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis());
|
||||
// poolConfig.setNumTestsPerEvictionRun(3);
|
||||
poolConfig.setBlockWhenExhausted(true);
|
||||
|
||||
return poolConfig;
|
||||
}
|
||||
|
||||
private static List<edu.lu.uni.serval.MultipleThreadsParser.MessageFile> getMessageFiles(String gumTreeInput) {
|
||||
String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile
|
||||
File revFilesPath = new File(inputPath + "revFiles/");
|
||||
File[] revFiles = revFilesPath.listFiles(); // project folders
|
||||
List<edu.lu.uni.serval.MultipleThreadsParser.MessageFile> msgFiles = new ArrayList<>();
|
||||
if (revFiles.length >= 0) {
|
||||
for (File revFile : revFiles) {
|
||||
// if (revFile.getName().endsWith(".java")) {
|
||||
String fileName = revFile.getName();
|
||||
File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
|
||||
fileName = fileName.replace(".java", ".txt");
|
||||
File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file
|
||||
File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file
|
||||
edu.lu.uni.serval.MultipleThreadsParser.MessageFile msgFile = new edu.lu.uni.serval.MultipleThreadsParser.MessageFile(revFile, prevFile, diffentryFile);
|
||||
msgFile.setPositionFile(positionFile);
|
||||
msgFiles.add(msgFile);
|
||||
// }
|
||||
}
|
||||
|
||||
return msgFiles;
|
||||
}
|
||||
else{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// return msgFiles;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package edu.lu.uni.serval.FixPatternParser.violations;
|
||||
|
||||
import edu.lu.uni.serval.FixPatternParser.Parser;
|
||||
import edu.lu.uni.serval.FixPatternParser.cluster.Compare;
|
||||
import redis.clients.jedis.JedisPool;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
public class RunnableCompare implements Runnable {
|
||||
|
||||
private String name;
|
||||
|
||||
@@ -6,13 +6,12 @@ import akka.actor.UntypedActor;
|
||||
import akka.japi.Creator;
|
||||
import akka.routing.RoundRobinPool;
|
||||
|
||||
import edu.lu.uni.serval.FixPatternParser.violations.WorkMessage;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader.loadRedis;
|
||||
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis;
|
||||
|
||||
public class TreeActor extends UntypedActor {
|
||||
|
||||
|
||||
@@ -3,35 +3,14 @@ package edu.lu.uni.serval.FixPatternParser.violations;
|
||||
import akka.actor.Props;
|
||||
import akka.actor.UntypedActor;
|
||||
import akka.japi.Creator;
|
||||
import com.github.gumtreediff.actions.ActionGenerator;
|
||||
import com.github.gumtreediff.actions.model.Action;
|
||||
import com.github.gumtreediff.matchers.Matcher;
|
||||
import com.github.gumtreediff.matchers.Matchers;
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
import edu.lu.uni.serval.FixPatternParser.RunnableParser;
|
||||
import edu.lu.uni.serval.MultipleThreadsParser.MessageFile;
|
||||
|
||||
import edu.lu.uni.serval.FixPatternParser.violations.WorkMessage;
|
||||
import edu.lu.uni.serval.config.Configuration;
|
||||
import edu.lu.uni.serval.utils.FileHelper;
|
||||
import edu.lu.uni.serval.FixPatternParser.cluster.Compare;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.Jedis;
|
||||
import redis.clients.jedis.JedisPool;
|
||||
import redis.clients.jedis.JedisPoolConfig;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.*;
|
||||
|
||||
import static edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader.comparePairs;
|
||||
import static edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader.loadRedis;
|
||||
import static edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader.getSimpliedTree;
|
||||
|
||||
public class TreeWorker extends UntypedActor {
|
||||
private static Logger log = LoggerFactory.getLogger(TreeWorker.class);
|
||||
|
||||
Reference in New Issue
Block a user