parallelism selection
This commit is contained in:
fixminer
@@ -1,7 +0,0 @@
|
||||
package edu.lu.uni.serval.config;
|
||||
|
||||
public class Configuration {
|
||||
|
||||
public static final long SECONDS_TO_WAIT = 900L;
|
||||
|
||||
}
|
||||
@@ -39,23 +39,24 @@ public class Launcher {
|
||||
String chunk = appProps.getProperty("chunk","1.txt");
|
||||
String eDiffTimeout = appProps.getProperty("eDiffTimeout","900");
|
||||
String isBig = appProps.getProperty("isBigPair","true");
|
||||
String parallelism = appProps.getProperty("parallelism","FORKJOIN");
|
||||
boolean isBigPair = Boolean.parseBoolean(isBig);
|
||||
|
||||
String parameters = String.format("\nportInner %s " +
|
||||
"\nnumOfWorkers %s " +
|
||||
"\njobType %s \nport %s " +
|
||||
"\npythonPath %s \ndatasetPath %s" +
|
||||
"\npjName %s \nactionType %s \nthreshold %s \ncursor %s \neDiffTimeout %s \nisBigPair %s"
|
||||
, portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,eDiffTimeout,isBig);
|
||||
"\npjName %s \nactionType %s \nthreshold %s \ncursor %s \neDiffTimeout %s \nisBigPair %s \nparallelism %s"
|
||||
, portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,eDiffTimeout,isBig,parallelism);
|
||||
|
||||
log.info(parameters);
|
||||
|
||||
mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk,eDiffTimeout,isBigPair);
|
||||
mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk,eDiffTimeout,isBigPair,parallelism);
|
||||
|
||||
|
||||
}
|
||||
|
||||
public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk, String eDiffTimeout, boolean isBigPair){
|
||||
public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk, String eDiffTimeout, boolean isBigPair, String parallelism){
|
||||
|
||||
|
||||
String dbDir;
|
||||
@@ -74,17 +75,17 @@ public class Launcher {
|
||||
try {
|
||||
switch (jobType) {
|
||||
case "ENHANCEDASTDIFF":
|
||||
EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName, eDiffTimeout,actionType);
|
||||
EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName, eDiffTimeout,actionType,parallelism);
|
||||
break;
|
||||
case "CACHE":
|
||||
StoreEDiffInCache.main(gumOutput, portDumps, dbDir, actionType+dumpsName,actionType);
|
||||
break;
|
||||
case "SI":
|
||||
CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType,isBigPair,iCursor);
|
||||
ImportPairs2DB.main(pairsPath+actionType, portInner, dbDir,datasetPath);
|
||||
ImportPairs2DB.main(pairsPath+actionType, portInner, dbDir,datasetPath,chunk);
|
||||
break;
|
||||
case "SIMI":
|
||||
AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,iCursor,chunk,eDiffTimeout);
|
||||
AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,iCursor,chunk,eDiffTimeout,parallelism);
|
||||
break;
|
||||
|
||||
case "LEVEL1":
|
||||
|
||||
@@ -1,600 +0,0 @@
|
||||
package edu.lu.uni.serval.fixminer;
|
||||
|
||||
import com.github.gumtreediff.actions.ActionGenerator;
|
||||
import com.github.gumtreediff.actions.model.Action;
|
||||
import com.github.gumtreediff.matchers.Matcher;
|
||||
import com.github.gumtreediff.matchers.Matchers;
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
import com.github.gumtreediff.tree.Tree;
|
||||
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
|
||||
import edu.lu.uni.serval.utils.ASTNodeMap;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.*;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* Created by anilkoyuncu on 19/03/2018.
|
||||
*/
|
||||
public class MultiThreadTreeLoader {
|
||||
|
||||
private static class StreamGobbler implements Runnable {
|
||||
private InputStream inputStream;
|
||||
private Consumer<String> consumer;
|
||||
|
||||
public StreamGobbler(InputStream inputStream, Consumer<String> consumer) {
|
||||
this.inputStream = inputStream;
|
||||
this.consumer = consumer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
new BufferedReader(new InputStreamReader(inputStream)).lines()
|
||||
.forEach(consumer);
|
||||
}
|
||||
}
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoader.class);
|
||||
|
||||
|
||||
public static void loadRedis(String cmd,String serverWait){
|
||||
Process process;
|
||||
|
||||
try {
|
||||
// String comd = String.format(cmd, f.getAbsoluteFile());
|
||||
process = Runtime.getRuntime()
|
||||
|
||||
.exec(cmd);
|
||||
|
||||
|
||||
StreamGobbler streamGobbler =
|
||||
new StreamGobbler(process.getInputStream(), System.out::println);
|
||||
Executors.newSingleThreadExecutor().submit(streamGobbler);
|
||||
// int exitCode = process.waitFor();
|
||||
// assert exitCode == 0;
|
||||
Thread.sleep(Integer.valueOf(serverWait));
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
log.info("Load done");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
|
||||
String inputPath;
|
||||
String outputPath;
|
||||
String port;
|
||||
String portInner;
|
||||
String pairsCSVPath;
|
||||
String importScript;
|
||||
String pairsCompletedPath;
|
||||
String serverWait;
|
||||
String option;
|
||||
String dbDir;
|
||||
String chunkName;
|
||||
if (args.length > 0) {
|
||||
inputPath = args[0];
|
||||
port = args[1];
|
||||
portInner = args[2];
|
||||
serverWait = args[3];
|
||||
option = args[4];
|
||||
chunkName = args[5];
|
||||
dbDir = args[6];
|
||||
// pairsCSVPath = args[3];
|
||||
// importScript = args[4];
|
||||
// pairsCompletedPath = args[3];
|
||||
} else {
|
||||
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
|
||||
outputPath = "/Users/anilkoyuncu/bugStudy/dataset/";
|
||||
port = "6379";
|
||||
portInner = "6380";
|
||||
serverWait = "10000";
|
||||
option = "COMP";
|
||||
pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test";
|
||||
importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh";
|
||||
pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed";
|
||||
chunkName ="chunk5.rdb";
|
||||
dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
|
||||
}
|
||||
|
||||
if (option.equals("CALC")) {
|
||||
calculatePairs(inputPath, port);
|
||||
log.info("Calculate pairs done");
|
||||
}else {
|
||||
comparePairs(port, inputPath, portInner, serverWait,chunkName,dbDir);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static void comparePairs(String port,String inputPath, String innerPort,String serverWait,String chunkName, String dbDir){
|
||||
// String cmd;
|
||||
// cmd = "bash " + importScript +" %s";
|
||||
|
||||
|
||||
|
||||
List<String> dir;
|
||||
List<String> path;
|
||||
|
||||
String orgDbname;
|
||||
|
||||
|
||||
File files = new File(dbDir);
|
||||
File[] listFiles = files.listFiles();
|
||||
|
||||
|
||||
Stream<File> stream = Arrays.stream(listFiles);
|
||||
List<File> folders = stream
|
||||
.filter(x -> x.getName().equals(chunkName))
|
||||
.collect(Collectors.toList());
|
||||
for (File folder : folders) {
|
||||
|
||||
|
||||
|
||||
|
||||
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
|
||||
cmd = String.format(cmd, dbDir,folder.getName(),Integer.valueOf(innerPort));
|
||||
loadRedis(cmd,serverWait);
|
||||
|
||||
|
||||
JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000);
|
||||
ScanResult<String> scan;
|
||||
try (Jedis inner = pool.getResource()) {
|
||||
while (inner.ping()== "PONG"){
|
||||
log.info("wait");
|
||||
}
|
||||
|
||||
ScanParams sc = new ScanParams();
|
||||
sc.count(150000000);
|
||||
sc.match("pair_*");
|
||||
|
||||
scan = inner.scan("0", sc);
|
||||
int size = scan.getResult().size();
|
||||
log.info("Scanning " + String.valueOf(size));
|
||||
}
|
||||
scan.getResult().parallelStream()
|
||||
.forEach(m -> coreCompare(m, inputPath, innerPort));
|
||||
|
||||
|
||||
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
|
||||
stopServer = String.format(stopServer,Integer.valueOf(innerPort));
|
||||
loadRedis(stopServer,serverWait);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
private static void coreCompare(String name , String inputPath, String innerPort) {
|
||||
JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000);
|
||||
Map<String, String> resultMap;
|
||||
try (Jedis jedis = pool.getResource()) {
|
||||
resultMap = jedis.hgetAll(name);
|
||||
}
|
||||
String[] split = name.split("_");
|
||||
|
||||
|
||||
String i = split[1];
|
||||
String j = split[2];
|
||||
String firstValue = resultMap.get("0");
|
||||
String secondValue = resultMap.get("1");
|
||||
|
||||
String[] firstValueSplit = firstValue.split("GumTreeOutput2");
|
||||
String[] secondValueSplit = secondValue.split("GumTreeOutput2");
|
||||
|
||||
if (firstValueSplit.length == 1) {
|
||||
firstValue = inputPath + firstValueSplit[0];
|
||||
} else {
|
||||
firstValue = inputPath + firstValueSplit[1];
|
||||
}
|
||||
|
||||
if (secondValueSplit.length == 1) {
|
||||
secondValue = inputPath + secondValueSplit[0];
|
||||
} else {
|
||||
secondValue = inputPath + secondValueSplit[1];
|
||||
}
|
||||
|
||||
try {
|
||||
ITree oldTree = getSimpliedTree(firstValue);
|
||||
|
||||
ITree newTree = getSimpliedTree(secondValue);
|
||||
|
||||
Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
|
||||
m.match();
|
||||
|
||||
|
||||
ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings());
|
||||
ag.generate();
|
||||
List<Action> actions = ag.getActions();
|
||||
|
||||
double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree);
|
||||
String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1);
|
||||
double diceSimilarity1 = m.diceSimilarity(oldTree, newTree);
|
||||
String diceSimilarity = String.format("%1.2f", diceSimilarity1);
|
||||
double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree);
|
||||
String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1);
|
||||
|
||||
String editDistance = String.valueOf(actions.size());
|
||||
|
||||
String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
|
||||
|
||||
|
||||
if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
|
||||
|| ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) {
|
||||
String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j);
|
||||
|
||||
try (Jedis jedis = pool.getResource()) {
|
||||
jedis.select(1);
|
||||
jedis.set(matchKey, result);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
try (Jedis jedis = pool.getResource()) {
|
||||
jedis.del("pair_"+ (String.valueOf(i)) + "_" + String.valueOf(j));
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}catch (Exception e){
|
||||
log.warn(e.toString() + " {}",(name));
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static void calculatePairs(String inputPath,String port) {
|
||||
File folder = new File(inputPath);
|
||||
File[] listOfFiles = folder.listFiles();
|
||||
Stream<File> stream = Arrays.stream(listOfFiles);
|
||||
List<File> pjs = stream
|
||||
.filter(x -> !x.getName().startsWith("."))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
List<File> fileToCompare = new ArrayList<>();
|
||||
for (File pj : pjs) {
|
||||
File[] files = pj.listFiles(new FilenameFilter() {
|
||||
public boolean accept(File dir, String name) {
|
||||
return name.startsWith("ActionSetDumps");
|
||||
}
|
||||
});
|
||||
Collections.addAll(fileToCompare, files[0].listFiles());
|
||||
|
||||
}
|
||||
System.out.println("a");
|
||||
// compareAll(fileToCompare);
|
||||
readMessageFiles(fileToCompare,port);
|
||||
}
|
||||
|
||||
// public static void processMessages(String inputPath, String outputPath) {
|
||||
// File folder = new File(outputPath + "pairs_splitted/");
|
||||
// File[] listOfFiles = folder.listFiles();
|
||||
// Stream<File> stream = Arrays.stream(listOfFiles);
|
||||
// List<File> pjs = stream
|
||||
// .filter(x -> !x.getName().startsWith("."))
|
||||
// .collect(Collectors.toList());
|
||||
// FileHelper.createDirectory(outputPath + "comparison_splitted/");
|
||||
// pjs.parallelStream()
|
||||
// .forEach(m -> coreLoop(m, outputPath,inputPath));
|
||||
// }
|
||||
|
||||
|
||||
public static ITree getSimpliedTree(String fn) {
|
||||
HierarchicalActionSet actionSet = null;
|
||||
try {
|
||||
FileInputStream fi = new FileInputStream(new File(fn));
|
||||
ObjectInputStream oi = new ObjectInputStream(fi);
|
||||
actionSet = (HierarchicalActionSet) oi.readObject();
|
||||
oi.close();
|
||||
fi.close();
|
||||
|
||||
|
||||
} catch (FileNotFoundException e) {
|
||||
log.error("File not found");
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
log.error("Error initializing stream");
|
||||
e.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
ITree parent = null;
|
||||
ITree children =null;
|
||||
ITree tree = getASTTree(actionSet, parent, children);
|
||||
tree.setParent(null);
|
||||
|
||||
return tree;
|
||||
|
||||
}
|
||||
|
||||
public static <T, E> List<T> getKeysByValue(Map<T, E> map, E value) {
|
||||
return map.entrySet()
|
||||
.stream()
|
||||
.filter(entry -> Objects.equals(entry.getValue(), value))
|
||||
.map(Map.Entry::getKey)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children){
|
||||
|
||||
int newType = 0;
|
||||
|
||||
String astNodeType = actionSet.getAstNodeType();
|
||||
List<Integer> keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType);
|
||||
|
||||
if(keysByValue.size() != 1){
|
||||
log.error("Birden cok astnodemapmapping");
|
||||
}
|
||||
newType = keysByValue.get(0);
|
||||
if(actionSet.getParent() == null){
|
||||
//root
|
||||
|
||||
parent = new Tree(newType,"");
|
||||
}else{
|
||||
children = new Tree(newType,"");
|
||||
parent.addChild(children);
|
||||
}
|
||||
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
|
||||
if (subActions.size() != 0){
|
||||
for (HierarchicalActionSet subAction : subActions) {
|
||||
|
||||
if(actionSet.getParent() == null){
|
||||
children = parent;
|
||||
}
|
||||
getASTTree(subAction,children,null);
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
return parent;
|
||||
}
|
||||
|
||||
// public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children){
|
||||
//
|
||||
// int newType = 0;
|
||||
//
|
||||
// Action action = actionSet.getAction();
|
||||
// if (action instanceof Update){
|
||||
// newType = 101;
|
||||
// }else if(action instanceof Insert){
|
||||
// newType =100;
|
||||
// }else if(action instanceof Move){
|
||||
// newType = 102;
|
||||
// }else if(action instanceof Delete){
|
||||
// newType=103;
|
||||
// }else{
|
||||
// new Exception("unknow action");
|
||||
// }
|
||||
// if(actionSet.getParent() == null){
|
||||
// //root
|
||||
//
|
||||
// parent = new Tree(newType,"");
|
||||
// }else{
|
||||
// children = new Tree(newType,"");
|
||||
// parent.addChild(children);
|
||||
// }
|
||||
// List<HierarchicalActionSet> subActions = actionSet.getSubActions();
|
||||
// if (subActions.size() != 0){
|
||||
// for (HierarchicalActionSet subAction : subActions) {
|
||||
//
|
||||
// if(actionSet.getParent() == null){
|
||||
// children = parent;
|
||||
// }
|
||||
// getActionTree(subAction,children,null);
|
||||
//
|
||||
// }
|
||||
//
|
||||
//
|
||||
// }
|
||||
// return parent;
|
||||
// }
|
||||
|
||||
|
||||
|
||||
private static void coreLoop(File mes, String outputPath,String inputPath) {
|
||||
try {
|
||||
|
||||
log.info("Starting in coreLoop");
|
||||
|
||||
BufferedReader br = null;
|
||||
String sCurrentLine = null;
|
||||
BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "comparison_splitted/" + "output_" + mes.getName()));
|
||||
|
||||
br = new BufferedReader(
|
||||
new FileReader(mes));
|
||||
while ((sCurrentLine = br.readLine()) != null) {
|
||||
String currentLine = sCurrentLine;
|
||||
String[] split = currentLine.split("\t");
|
||||
String i = split[0];
|
||||
String j = split[1];
|
||||
String firstValue = split[2];
|
||||
String secondValue = split[3];
|
||||
|
||||
firstValue = inputPath + firstValue.split("GumTreeOutput2")[1];
|
||||
secondValue = inputPath + secondValue.split("GumTreeOutput2")[1];
|
||||
|
||||
ITree oldTree = getSimpliedTree(firstValue);
|
||||
|
||||
ITree newTree = getSimpliedTree(secondValue);
|
||||
|
||||
Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
|
||||
m.match();
|
||||
|
||||
ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings());
|
||||
ag.generate();
|
||||
List<Action> actions = ag.getActions();
|
||||
writer.write(String.valueOf(i));
|
||||
writer.write("\t");
|
||||
writer.write(String.valueOf(j));
|
||||
writer.write("\t");
|
||||
|
||||
writer.write(String.format("%1.2f", m.chawatheSimilarity(oldTree, newTree)));
|
||||
writer.write("\t");
|
||||
writer.write(String.format("%1.2f", m.diceSimilarity(oldTree, newTree)));
|
||||
writer.write("\t");
|
||||
writer.write(String.format("%1.2f", m.jaccardSimilarity(oldTree, newTree)));
|
||||
writer.write("\t");
|
||||
writer.write(String.valueOf(actions.size()));
|
||||
writer.write("\t");
|
||||
writer.write(firstValue);
|
||||
writer.write("\t");
|
||||
writer.write(secondValue);
|
||||
writer.write("\n");
|
||||
|
||||
|
||||
}
|
||||
writer.close();
|
||||
} catch (FileNotFoundException e) {
|
||||
log.error("File not found");
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
log.error("Error initializing stream");
|
||||
e.printStackTrace();
|
||||
|
||||
}
|
||||
log.info("Completed output_" + mes.getName());
|
||||
}
|
||||
|
||||
private static void readMessageFiles(List<File> folders,String port) {
|
||||
|
||||
List<String> treesFileNames = new ArrayList<>();
|
||||
|
||||
|
||||
for (File target : folders) {
|
||||
|
||||
treesFileNames.add(target.toString());
|
||||
}
|
||||
// FileHelper.createDirectory(outputPath + "pairs/");
|
||||
log.info("Calculating pairs");
|
||||
// treesFileNames = treesFileNames.subList(0,100);
|
||||
byte [] buf = new byte[0];
|
||||
String line = null;
|
||||
|
||||
|
||||
// FileChannel rwChannel = new RandomAccessFile(outputPath + "pairs/" +"textfile.txt", "rw").getChannel();
|
||||
// ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE);
|
||||
int fileCounter = 0;
|
||||
|
||||
JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
|
||||
try (Jedis jedis = jedisPool.getResource()) {
|
||||
List<String> dir = null;
|
||||
List<String> path = null;
|
||||
for (int i = 0; i < treesFileNames.size(); i++) {
|
||||
for (int j = i + 1; j < treesFileNames.size(); j++) {
|
||||
|
||||
|
||||
// do operations with jedis resource
|
||||
|
||||
String key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j);
|
||||
// String value = treesFileNames.get(i).split("GumTreeOutput2")[1] +","+treesFileNames.get(j).split("GumTreeOutput2")[1];
|
||||
// jedis.set(key,value);
|
||||
|
||||
jedis.hset(key,"0",treesFileNames.get(i).split("GumTreeOutput2")[1]);
|
||||
jedis.hset(key,"1",treesFileNames.get(j).split("GumTreeOutput2")[1]);
|
||||
//10000000
|
||||
if(Integer.compare(jedis.dbSize().intValue(),10000000) == 0){
|
||||
dir = jedis.configGet("dir");
|
||||
path = jedis.configGet("dbfilename");
|
||||
File dbPath = new File(dir.get(1)+"/"+path.get(1));
|
||||
File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb");
|
||||
try {
|
||||
jedis.save();
|
||||
while (jedis.ping()== "PONG"){
|
||||
log.info("wait");
|
||||
}
|
||||
|
||||
|
||||
|
||||
Files.copy(dbPath.toPath(),savePath.toPath());
|
||||
} catch (IOException e) {
|
||||
|
||||
e.printStackTrace();
|
||||
}
|
||||
fileCounter++;
|
||||
jedis.flushDB();
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
jedis.save();
|
||||
fileCounter++;
|
||||
File dbPath = new File(dir.get(1)+"/"+path.get(1));
|
||||
File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb");
|
||||
try {
|
||||
|
||||
while (jedis.ping()== "PONG"){
|
||||
log.info("wait");
|
||||
}
|
||||
|
||||
|
||||
|
||||
Files.copy(dbPath.toPath(),savePath.toPath());
|
||||
} catch (IOException e) {
|
||||
|
||||
e.printStackTrace();
|
||||
}
|
||||
jedis.flushDB();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
log.info("Done pairs");
|
||||
}
|
||||
|
||||
static final JedisPoolConfig poolConfig = buildPoolConfig();
|
||||
|
||||
|
||||
private static JedisPoolConfig buildPoolConfig() {
|
||||
final JedisPoolConfig poolConfig = new JedisPoolConfig();
|
||||
poolConfig.setMaxTotal(128);
|
||||
poolConfig.setMaxIdle(128);
|
||||
poolConfig.setMinIdle(16);
|
||||
poolConfig.setTestOnBorrow(true);
|
||||
poolConfig.setTestOnReturn(true);
|
||||
poolConfig.setTestWhileIdle(true);
|
||||
poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis());
|
||||
poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis());
|
||||
poolConfig.setNumTestsPerEvictionRun(3);
|
||||
poolConfig.setBlockWhenExhausted(true);
|
||||
|
||||
return poolConfig;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// return msgFiles;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,239 +0,0 @@
|
||||
package edu.lu.uni.serval.fixminer;
|
||||
|
||||
import akka.actor.ActorRef;
|
||||
import akka.actor.ActorSystem;
|
||||
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor;
|
||||
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage;
|
||||
import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile;
|
||||
import edu.lu.uni.serval.utils.FileHelper;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public class TestHunkParserSingleFile {
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(TestHunkParserSingleFile.class);
|
||||
// public static void main(String[] args) {
|
||||
// // input data
|
||||
//
|
||||
//// String rootPath = "/Users/anilkoyuncu/bugStudy";
|
||||
// String inputPath;
|
||||
// String outputPath;
|
||||
// if(args.length > 0){
|
||||
// inputPath = args[1];
|
||||
// outputPath = args[0];
|
||||
// }else{
|
||||
//// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeInputBug4";
|
||||
// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/allDataset";
|
||||
//// outputPath = "/Users/anilkoyuncu/bugStudy/code/python/GumTreeOutput2/";
|
||||
// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputSingle";
|
||||
// }
|
||||
//
|
||||
// //5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.txt_1_CAMEL
|
||||
//
|
||||
// File folder = new File(inputPath);
|
||||
// File[] listOfFiles = folder.listFiles();
|
||||
// Stream<File> stream = Arrays.stream(listOfFiles);
|
||||
// List<File> folders = stream
|
||||
// .filter(x -> !x.getName().startsWith("."))
|
||||
//
|
||||
// .collect(Collectors.toList());
|
||||
//
|
||||
//// List<File> targetList = new ArrayList<File>();
|
||||
//// for (File f:folders){
|
||||
//// for(File f1 :f.listFiles()){
|
||||
//// if (!f1.getName().startsWith(".")){
|
||||
//// targetList.add(f1);
|
||||
//// }
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
//// List<String> pjList = Arrays.asList("DATAJPA","ZXing","PDE","SWS","SWT", "SWF", "COLLECTIONS", "JDT");
|
||||
// List<String> files = new ArrayList<String>();
|
||||
//// files.add("5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.java");
|
||||
// files.add("d6c5e5_9f96d9_hbase-server#src#main#java#org#apache#hadoop#hbase#master#RegionStates.java");
|
||||
// for(String f : files){
|
||||
// String pjName = "HBASE";
|
||||
//// for (File target : folders) {
|
||||
//// String pjName = target.getName();
|
||||
//// if (!pjList.contains(pjName)){
|
||||
//// continue;
|
||||
//// }
|
||||
//
|
||||
//// final List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
|
||||
// MessageFile messageFile = getMessageFile(inputPath + "/" + pjName +"/", f);
|
||||
//
|
||||
// List<MessageFile> msgFiles = new ArrayList<>();
|
||||
// msgFiles.add(messageFile);
|
||||
// String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/";
|
||||
// final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list";
|
||||
// final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list";
|
||||
// final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list";
|
||||
// final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv";
|
||||
// final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list";
|
||||
//
|
||||
//
|
||||
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ActionSetDumps");
|
||||
// FileHelper.deleteDirectory(editScriptsFilePath);
|
||||
// FileHelper.deleteDirectory(patchesSourceCodeFilePath);
|
||||
// FileHelper.deleteDirectory(buggyTokensFilePath);
|
||||
// FileHelper.deleteDirectory(editScriptSizesFilePath);
|
||||
// FileHelper.deleteDirectory(alarmTypesFilePath);
|
||||
//
|
||||
// StringBuilder astEditScripts = new StringBuilder();
|
||||
// StringBuilder tokens = new StringBuilder();
|
||||
// StringBuilder sizes = new StringBuilder();
|
||||
// StringBuilder patches = new StringBuilder();
|
||||
// StringBuilder alarmTypes = new StringBuilder();
|
||||
//
|
||||
//// int a = 0;
|
||||
//
|
||||
// ActorSystem system = null;
|
||||
// ActorRef parsingActor = null;
|
||||
// final EDiffMessage msg = new EDiffMessage(0, msgFiles);
|
||||
// try {
|
||||
// log.info("Akka begins...");
|
||||
// system = ActorSystem.create("Mining-FixPattern-System");
|
||||
// parsingActor = system.actorOf(EDiffActor.props(1, "dataset"), "mine-fix-pattern-actor");
|
||||
// parsingActor.tell(msg, ActorRef.noSender());
|
||||
// } catch (Exception e) {
|
||||
// system.shutdown();
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
//
|
||||
//// int counter = 0;
|
||||
//// for (MessageFile msgFile : msgFiles) {
|
||||
//// EDiffHunkParser parser = new EDiffHunkParser();
|
||||
////
|
||||
//// final ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
//// // schedule the work
|
||||
//// final Future<?> future = executor.submit(new RunnableParser(msgFile.getPrevFile(),
|
||||
//// msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser));
|
||||
//// try {
|
||||
//// // where we wait for task to complete
|
||||
//// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS);
|
||||
//// String editScripts = parser.getAstEditScripts();
|
||||
//// if (!editScripts.equals("")) {
|
||||
//// astEditScripts.append(editScripts);
|
||||
//// tokens.append(parser.getTokensOfSourceCode());
|
||||
//// sizes.append(parser.getSizes());
|
||||
//// patches.append(parser.getPatchesSourceCode());
|
||||
//// alarmTypes.append(parser.getAlarmTypes());
|
||||
////
|
||||
//// a++;
|
||||
//// if (a % 100 == 0) {
|
||||
//// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true);
|
||||
//// FileHelper.outputToFile(buggyTokensFilePath, tokens, true);
|
||||
//// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true);
|
||||
//// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true);
|
||||
//// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true);
|
||||
//// astEditScripts.setLength(0);
|
||||
//// tokens.setLength(0);
|
||||
//// sizes.setLength(0);
|
||||
//// patches.setLength(0);
|
||||
//// alarmTypes.setLength(0);
|
||||
//// System.out.println("Finish of parsing " + a + " files......");
|
||||
//// }
|
||||
//// }
|
||||
//// } catch (TimeoutException e) {
|
||||
//// err.println("task timed out");
|
||||
//// future.cancel(true /* mayInterruptIfRunning */);
|
||||
//// } catch (InterruptedException e) {
|
||||
//// err.println("task interrupted");
|
||||
//// } catch (ExecutionException e) {
|
||||
//// err.println("task aborted");
|
||||
//// } finally {
|
||||
//// executor.shutdownNow();
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true);
|
||||
// FileHelper.outputToFile(buggyTokensFilePath, tokens, true);
|
||||
// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true);
|
||||
// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true);
|
||||
// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true);
|
||||
// astEditScripts.setLength(0);
|
||||
// tokens.setLength(0);
|
||||
// sizes.setLength(0);
|
||||
// patches.setLength(0);
|
||||
// alarmTypes.setLength(0);
|
||||
//
|
||||
//
|
||||
//// classifyByAlarmTypes();
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
private static List<MessageFile> getMessageFiles(String gumTreeInput) {
|
||||
String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile
|
||||
File revFilesPath = new File(inputPath + "revFiles/");
|
||||
File[] revFiles = revFilesPath.listFiles(); // project folders
|
||||
List<MessageFile> msgFiles = new ArrayList<>();
|
||||
|
||||
// gumTreeInput = /Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/AMQP/
|
||||
// fileName = 01534a_df5570_spring-rabbit#src#test#java#org#springframework#amqp#rabbit#listener#LocallyTransactedTests.java
|
||||
if (revFiles.length >= 0) {
|
||||
for (File revFile : revFiles) {
|
||||
// if (revFile.getName().endsWith(".java")) {
|
||||
String fileName = revFile.getName();
|
||||
File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
|
||||
fileName = fileName.replace(".java", ".txt");
|
||||
File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file
|
||||
File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file
|
||||
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
|
||||
msgFile.setPositionFile(positionFile);
|
||||
msgFiles.add(msgFile);
|
||||
// }
|
||||
}
|
||||
|
||||
return msgFiles;
|
||||
}
|
||||
else{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static MessageFile getMessageFile(String gumTreeInput, String fileName) {
|
||||
// String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile
|
||||
// File revFilesPath = new File(inputPath + "revFiles/");
|
||||
// File[] revFiles = revFilesPath.listFiles(); // project folders
|
||||
// List<MessageFile> msgFiles = new ArrayList<>();
|
||||
|
||||
// gumTreeInput = /Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/AMQP/
|
||||
// fileName = 01534a_df5570_spring-rabbit#src#test#java#org#springframework#amqp#rabbit#listener#LocallyTransactedTests.java
|
||||
// if (revFiles.length >= 0) {
|
||||
// for (File revFile : revFiles) {
|
||||
// if (revFile.getName().endsWith(".java")) {
|
||||
// String fileName = revFile.getName();
|
||||
File revFile = new File(gumTreeInput + "revFiles/"+fileName);
|
||||
File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
|
||||
fileName = fileName.replace(".java", ".txt");
|
||||
File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file
|
||||
// File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file
|
||||
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
|
||||
return msgFile;
|
||||
// msgFile.setPositionFile(positionFile);
|
||||
// msgFiles.add(msgFile);
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// return msgFiles;
|
||||
// }
|
||||
// else{
|
||||
// return null;
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -19,26 +19,57 @@ public class AkkaTreeParser {
|
||||
private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class);
|
||||
|
||||
|
||||
public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, int cursor, String eDiffTimeout){
|
||||
public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, int cursor, String eDiffTimeout, String parallelism){
|
||||
|
||||
final List<String> listOfPairs = getMessages(innerPool,cursor); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
|
||||
|
||||
|
||||
switch (parallelism){
|
||||
case "AKKA":
|
||||
ActorSystem system = null;
|
||||
ActorRef parsingActor = null;
|
||||
final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,eDiffTimeout);
|
||||
try {
|
||||
log.info("Akka begins...");
|
||||
system = ActorSystem.create("Compare-EnhancedDiff-System");
|
||||
|
||||
parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor");
|
||||
parsingActor.tell(msg, ActorRef.noSender());
|
||||
} catch (Exception e) {
|
||||
system.shutdown();
|
||||
e.printStackTrace();
|
||||
}
|
||||
break;
|
||||
case "FORKJOIN":
|
||||
int counter = new Object() {
|
||||
int counter = 0;
|
||||
|
||||
ActorSystem system = null;
|
||||
ActorRef parsingActor = null;
|
||||
final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,eDiffTimeout);
|
||||
try {
|
||||
log.info("Akka begins...");
|
||||
system = ActorSystem.create("Compare-EnhancedDiff-System");
|
||||
|
||||
parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor");
|
||||
parsingActor.tell(msg, ActorRef.noSender());
|
||||
} catch (Exception e) {
|
||||
system.shutdown();
|
||||
e.printStackTrace();
|
||||
{
|
||||
listOfPairs.stream().
|
||||
parallel().
|
||||
peek(x -> counter++).
|
||||
forEach(m ->
|
||||
{
|
||||
Compare compare = new Compare();
|
||||
compare.coreCompare(m, innerPool, outerPool);
|
||||
if (counter % 10 == 0) {
|
||||
log.info("Finalized parsing " + counter + " files... remaing " + (listOfPairs.size() - counter));
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
}.counter;
|
||||
log.info("Finished parsing {} files",counter);
|
||||
break;
|
||||
default:
|
||||
log.error("Unknown parallelism {}", parallelism);
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
public static List<String> getMessages(JedisPool innerPool, int cursor){
|
||||
|
||||
@@ -5,7 +5,7 @@ import com.github.gumtreediff.actions.model.Action;
|
||||
import com.github.gumtreediff.matchers.Matcher;
|
||||
import com.github.gumtreediff.matchers.Matchers;
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
import edu.lu.uni.serval.utils.EDiff;
|
||||
import edu.lu.uni.serval.utils.EDiffHelper;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.Jedis;
|
||||
@@ -32,7 +32,6 @@ public class Compare {
|
||||
|
||||
try {
|
||||
jedis = innerPool.getResource();
|
||||
// resultMap = jedis.hgetAll(name);
|
||||
|
||||
String[] split = name.split("_");
|
||||
|
||||
@@ -41,12 +40,9 @@ public class Compare {
|
||||
String j = split[2];
|
||||
|
||||
|
||||
// String firstValue = resultMap.get("0");
|
||||
// String secondValue = resultMap.get("1");
|
||||
oldTree = EDiffHelper.getSimpliedTree(i,outerPool);
|
||||
|
||||
oldTree = EDiff.getSimpliedTree(i,outerPool);
|
||||
|
||||
newTree = EDiff.getSimpliedTree(j,outerPool);
|
||||
newTree = EDiffHelper.getSimpliedTree(j,outerPool);
|
||||
|
||||
Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
|
||||
m.match();
|
||||
|
||||
@@ -62,13 +62,13 @@ public class TreeActor extends UntypedActor {
|
||||
List<String> pairsOfWorkers = pairs.subList(fromIndex, toIndex);
|
||||
final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool,((TreeMessage) message).getSECONDS_TO_WAIT());
|
||||
mineRouter.tell(workMsg, getSelf());
|
||||
logger.info("Assign a task to worker #" + (i + 1) + "...");
|
||||
logger.info("Assign {} task to worker #" + (i + 1) + "...",pairsOfWorkers.size());
|
||||
}
|
||||
} else if ("STOP".equals(message.toString())) {
|
||||
counter ++;
|
||||
logger.info(counter + " workers finailized their work...");
|
||||
logger.info(counter + " workers finalized their work...");
|
||||
if (counter >= numberOfWorkers) {
|
||||
logger.info("All workers finailized their work...");
|
||||
logger.info("All workers finalized their work...");
|
||||
this.getContext().stop(mineRouter);
|
||||
this.getContext().stop(getSelf());
|
||||
this.getContext().system().shutdown();
|
||||
|
||||
@@ -3,7 +3,6 @@ package edu.lu.uni.serval.fixminer.akka.compare;
|
||||
import akka.actor.Props;
|
||||
import akka.actor.UntypedActor;
|
||||
import akka.japi.Creator;
|
||||
import edu.lu.uni.serval.config.Configuration;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.JedisPool;
|
||||
@@ -15,32 +14,7 @@ public class TreeWorker extends UntypedActor {
|
||||
private static Logger log = LoggerFactory.getLogger(TreeWorker.class);
|
||||
|
||||
|
||||
// private JedisPool innerPool;
|
||||
// private JedisPool outerPool;
|
||||
//
|
||||
// public TreeWorker(String innerPort,String outerPort) {
|
||||
//// this.innerPool = innerPool;
|
||||
//// this.outerPool = outerPool;
|
||||
// this.outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(outerPort),20000000);
|
||||
// this.innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(innerPort),20000000);
|
||||
//
|
||||
//
|
||||
// }
|
||||
//
|
||||
// public static Props props(final String innerPort,final String outerPort) {
|
||||
// return Props.create(new Creator<TreeWorker>() {
|
||||
//
|
||||
// private static final long serialVersionUID = -7615153844097275009L;
|
||||
//
|
||||
// @Override
|
||||
// public TreeWorker create() throws Exception {
|
||||
// return new TreeWorker(innerPort,outerPort);
|
||||
// }
|
||||
//
|
||||
// });
|
||||
// }
|
||||
|
||||
public TreeWorker() {
|
||||
public TreeWorker() {
|
||||
|
||||
}
|
||||
|
||||
@@ -63,160 +37,46 @@ public class TreeWorker extends UntypedActor {
|
||||
public void onReceive(Object message) throws Exception {
|
||||
if(message instanceof TreeMessage) {
|
||||
|
||||
|
||||
// if (message instanceof edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage) {
|
||||
TreeMessage msg = (TreeMessage) message;
|
||||
List<String> files = msg.getName();
|
||||
JedisPool innerPool = msg.getInnerPool();
|
||||
JedisPool outerPool = msg.getOuterPool();
|
||||
|
||||
int id = msg.getId();
|
||||
// int counter = new Object() {
|
||||
int counter = 0;
|
||||
int counter = 0;
|
||||
|
||||
//
|
||||
for (String name : files)
|
||||
{
|
||||
// files.stream().
|
||||
// parallel().
|
||||
// peek(x -> counter++).
|
||||
// forEach(m ->
|
||||
// {
|
||||
// Compare compare = new Compare();
|
||||
// compare.coreCompare(m, innerPool, outerPool);
|
||||
// }
|
||||
// );
|
||||
// }
|
||||
// }.counter;
|
||||
|
||||
//
|
||||
final ExecutorService executor = Executors.newWorkStealingPool();
|
||||
|
||||
Compare compare = new Compare();
|
||||
final ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
// // schedule the work
|
||||
final Future<?> future = executor.submit(new RunnableCompare(name, innerPool, outerPool, new Compare()));
|
||||
final Future<?> future = executor.submit(new RunnableCompare(name, innerPool, outerPool, compare));
|
||||
try {
|
||||
// wait for task to complete
|
||||
future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS);
|
||||
Compare compare = new Compare();
|
||||
compare.coreCompare(name, innerPool, outerPool);
|
||||
counter++;
|
||||
// nullDiffEntry += parser.nullMatchedDiffEntry;
|
||||
// nullMappingGumTreeResults += parser.nullMappingGumTreeResult;
|
||||
// pureDeletion += parser.pureDeletions;
|
||||
// largeHunk += parser.largeHunk;
|
||||
// nullSourceCode += parser.nullSourceCode;
|
||||
// testInfos += parser.testInfos;
|
||||
// testingInfo.append(parser.testingInfo);
|
||||
// builder.append(parser.unfixedViolations);
|
||||
//
|
||||
// String editScript = parser.getAstEditScripts();
|
||||
// if ("".equals(editScript)) {
|
||||
//// if (parser.resultType == 1) {
|
||||
//// nullGumTreeResults += countAlarms(positionFile, "");
|
||||
//// } else if (parser.resultType == 2) {
|
||||
//// noSourceCodeChanges += countAlarms(positionFile, "");
|
||||
//// } else if (parser.resultType == 3) {
|
||||
//// noStatementChanges += countAlarms(positionFile, "");
|
||||
//// }
|
||||
// } else {
|
||||
// editScripts.append(editScript);
|
||||
// patchesSourceCode.append(parser.getPatchesSourceCode());
|
||||
// sizes.append(parser.getSizes());
|
||||
// tokens.append(parser.getTokensOfSourceCode());
|
||||
//
|
||||
// counter ++;
|
||||
// if (counter % 100 == 0) {
|
||||
// FileHelper.outputToFile(editScriptsFilePath + "edistScripts_" + id + ".list", editScripts, true);
|
||||
// FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true);
|
||||
// FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true);
|
||||
// FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true);
|
||||
// editScripts.setLength(0);
|
||||
// patchesSourceCode.setLength(0);
|
||||
// sizes.setLength(0);
|
||||
// tokens.setLength(0);
|
||||
// log.info("Worker #" + id +" finialized parsing " + counter + " files...");
|
||||
// FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true);
|
||||
// testingInfo.setLength(0);
|
||||
// }
|
||||
// }
|
||||
if (counter % 100 == 0) {
|
||||
log.info("Worker #" + id +" finalized parsing " + counter + " pairs... remaing "+ (files.size() - counter));
|
||||
}
|
||||
} catch (TimeoutException e) {
|
||||
future.cancel(true);
|
||||
////// timeouts += countAlarms(positionFile, "#Timeout:");
|
||||
// System.err.println("#Timeout: " + name);
|
||||
} catch (InterruptedException e) {
|
||||
////// timeouts += countAlarms(positionFile, "#TimeInterrupted:");
|
||||
//// System.err.println("#TimeInterrupted: " + revFile.getName());
|
||||
e.printStackTrace();
|
||||
} catch (ExecutionException e) {
|
||||
////// timeouts += countAlarms(positionFile, "#TimeAborted:");
|
||||
//// System.err.println("#TimeAborted: " + revFile.getName());
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
// log.info("Worker #" + id +"finialized parsing " + counter + " files...");
|
||||
log.info("Worker #" + id + " finalized the work...");
|
||||
this.getSender().tell("STOP", getSelf());
|
||||
// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
|
||||
// stopServer = String.format(stopServer,Integer.valueOf(innerPort));
|
||||
// loadRedis(stopServer,serverWait);
|
||||
//
|
||||
// if (sizes.length() > 0) {
|
||||
// FileHelper.outputToFile(editScriptsFilePath + "editScripts_" + id + ".list", editScripts, true);
|
||||
// FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true);
|
||||
// FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true);
|
||||
// FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true);
|
||||
// editScripts.setLength(0);
|
||||
// patchesSourceCode.setLength(0);
|
||||
// sizes.setLength(0);
|
||||
// tokens.setLength(0);
|
||||
//
|
||||
// FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true);
|
||||
// testingInfo.setLength(0);
|
||||
// }
|
||||
// String statistic = "\nNullGumTreeResults: " + nullGumTreeResults + "\nNoSourceCodeChanges: " + noSourceCodeChanges +
|
||||
// "\nNoStatementChanges: " + noStatementChanges + "\nNullDiffEntry: " + nullDiffEntry + "\nNullMatchedGumTreeResults: " + nullMappingGumTreeResults +
|
||||
// "\nPureDeletion: " + pureDeletion + "\nLargeHunk: " + largeHunk + "\nNullSourceCode: " + nullSourceCode +
|
||||
// "\nTestingInfo: " + testInfos + "\nTimeout: " + timeouts;
|
||||
// FileHelper.outputToFile("OUTPUT/statistic_" + id + ".list", statistic, false);
|
||||
// FileHelper.outputToFile("OUTPUT/UnfixedV_" + id + ".list", builder, false);
|
||||
//
|
||||
// log.info("Worker #" + id +"finialized parsing " + counter + " files...");
|
||||
// log.info("Worker #" + id + " finialized the work...");
|
||||
// this.getSender().tell("STOP", getSelf());
|
||||
// } else {
|
||||
}else{
|
||||
unhandled(message);
|
||||
}
|
||||
}
|
||||
|
||||
// static final JedisPoolConfig poolConfig = buildPoolConfig();
|
||||
//
|
||||
//
|
||||
// private static JedisPoolConfig buildPoolConfig() {
|
||||
// final JedisPoolConfig poolConfig = new JedisPoolConfig();
|
||||
// poolConfig.setMaxTotal(128);
|
||||
// poolConfig.setMaxIdle(128);
|
||||
// poolConfig.setMinIdle(16);
|
||||
// poolConfig.setTestOnBorrow(true);
|
||||
// poolConfig.setTestOnReturn(true);
|
||||
// poolConfig.setTestWhileIdle(true);
|
||||
// poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis());
|
||||
// poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis());
|
||||
// poolConfig.setNumTestsPerEvictionRun(3);
|
||||
// poolConfig.setBlockWhenExhausted(true);
|
||||
//
|
||||
// return poolConfig;
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -56,7 +56,7 @@ public class EDiffActor extends UntypedActor {
|
||||
List<MessageFile> filesOfWorkers = files.subList(fromIndex, toIndex);
|
||||
final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getActionType());
|
||||
mineRouter.tell(workMsg, getSelf());
|
||||
logger.info("Assign a task to worker #" + (i + 1) + "...");
|
||||
logger.info("Assign {} task to worker #" + (i + 1) ,filesOfWorkers.size());
|
||||
}
|
||||
} else if ("STOP".equals(message.toString())) {
|
||||
counter ++;
|
||||
|
||||
@@ -18,53 +18,45 @@ import java.util.List;
|
||||
*/
|
||||
public class EDiffHunkParser extends EDiffParser {
|
||||
|
||||
public String testingInfo = "";
|
||||
|
||||
public int nullMappingGumTreeResult = 0;
|
||||
public int pureDeletions = 0;
|
||||
public int largeHunk = 0;
|
||||
public int nullSourceCode = 0;
|
||||
public int nullMatchedDiffEntry = 0;
|
||||
public int testInfos = 0;
|
||||
|
||||
public String unfixedViolations = "";
|
||||
|
||||
@Override
|
||||
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project,String actionType) {
|
||||
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile);
|
||||
if (actionSets.size() != 0) {
|
||||
String folder= null;
|
||||
boolean processActionSet = false;
|
||||
switch (actionType){
|
||||
case "ALL":
|
||||
// switch (actionType){
|
||||
// case "ALL":
|
||||
if(actionType.equals("ALL")){
|
||||
folder = "/ALL/";
|
||||
processActionSet = true;
|
||||
break;
|
||||
case "UPD":
|
||||
processActionSet =
|
||||
actionSets.stream().allMatch(p -> p.getAction() instanceof Update);
|
||||
}else if(actionType.equals("UPD") || actionType.equals("INS") || actionType.equals("DEL") || actionType.equals("MOV")|| actionType.equals("MIX")){
|
||||
boolean isUPD = actionSets.stream().allMatch(p -> p.getAction() instanceof Update);
|
||||
boolean isINS = actionSets.stream().allMatch(p -> p.getAction() instanceof Insert);
|
||||
boolean isDEL = actionSets.stream().allMatch(p -> p.getAction() instanceof Delete);
|
||||
boolean isMOV = actionSets.stream().allMatch(p -> p.getAction() instanceof Move);
|
||||
if(isUPD){
|
||||
folder = "/UPD/";
|
||||
break;
|
||||
case "INS":
|
||||
processActionSet =
|
||||
actionSets.stream().allMatch(p -> p.getAction() instanceof Insert);
|
||||
|
||||
processActionSet = true;
|
||||
}else if(isINS){
|
||||
folder = "/INS/";
|
||||
break;
|
||||
case "DEL":
|
||||
processActionSet =
|
||||
actionSets.stream().allMatch(p -> p.getAction() instanceof Delete);
|
||||
processActionSet = true;
|
||||
}else if(isDEL){
|
||||
folder = "/DEL/";
|
||||
break;
|
||||
case "MOV":
|
||||
processActionSet =
|
||||
actionSets.stream().allMatch(p -> p.getAction() instanceof Move);
|
||||
processActionSet = true;
|
||||
}else if(isMOV){
|
||||
folder = "/MOV/";
|
||||
break;
|
||||
default:
|
||||
processActionSet = true;
|
||||
}else{
|
||||
folder = "/MIX/";
|
||||
processActionSet = true;
|
||||
}
|
||||
}else{
|
||||
|
||||
|
||||
processActionSet = false;
|
||||
System.err.print(actionType + "not known");
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -78,7 +70,6 @@ public class EDiffHunkParser extends EDiffParser {
|
||||
FileOutputStream f = null;
|
||||
|
||||
try {
|
||||
// String pj = diffentryFile.getParent().split("Defects4J")[1];
|
||||
String datasetName = project;
|
||||
String[] split1 = diffentryFile.getParent().split(datasetName);
|
||||
String root = split1[0];
|
||||
|
||||
@@ -38,27 +38,10 @@ public class EDiffWorker extends UntypedActor {
|
||||
if (message instanceof EDiffMessage) {
|
||||
EDiffMessage msg = (EDiffMessage) message;
|
||||
List<MessageFile> files = msg.getMsgFiles();
|
||||
StringBuilder editScripts = new StringBuilder();
|
||||
StringBuilder patchesSourceCode = new StringBuilder();
|
||||
StringBuilder sizes = new StringBuilder();
|
||||
StringBuilder tokens = new StringBuilder();
|
||||
StringBuilder testingInfo = new StringBuilder();
|
||||
|
||||
int id = msg.getId();
|
||||
int counter = 0;
|
||||
|
||||
int nullGumTreeResults = 0;
|
||||
int noSourceCodeChanges = 0;
|
||||
int noStatementChanges = 0;
|
||||
int nullDiffEntry = 0;
|
||||
int nullMappingGumTreeResults = 0;
|
||||
int pureDeletion = 0;
|
||||
int largeHunk = 0;
|
||||
int nullSourceCode = 0;
|
||||
int testInfos = 0;
|
||||
int timeouts = 0;
|
||||
StringBuilder builder = new StringBuilder();
|
||||
|
||||
for (MessageFile msgFile : files) {
|
||||
File revFile = msgFile.getRevFile();
|
||||
File prevFile = msgFile.getPrevFile();
|
||||
@@ -67,41 +50,17 @@ public class EDiffWorker extends UntypedActor {
|
||||
|
||||
|
||||
EDiffHunkParser parser = new EDiffHunkParser();
|
||||
|
||||
final ExecutorService executor = Executors.newWorkStealingPool();
|
||||
|
||||
final ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
// schedule the work
|
||||
final Future<?> future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getActionType()));
|
||||
try {
|
||||
// wait for task to complete
|
||||
future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS);
|
||||
|
||||
nullDiffEntry += parser.nullMatchedDiffEntry;
|
||||
nullMappingGumTreeResults += parser.nullMappingGumTreeResult;
|
||||
pureDeletion += parser.pureDeletions;
|
||||
largeHunk += parser.largeHunk;
|
||||
nullSourceCode += parser.nullSourceCode;
|
||||
testInfos += parser.testInfos;
|
||||
testingInfo.append(parser.testingInfo);
|
||||
builder.append(parser.unfixedViolations);
|
||||
|
||||
String editScript = parser.getAstEditScripts();
|
||||
if ("".equals(editScript)) {
|
||||
|
||||
} else {
|
||||
editScripts.append(editScript);
|
||||
patchesSourceCode.append(parser.getPatchesSourceCode());
|
||||
sizes.append(parser.getSizes());
|
||||
tokens.append(parser.getTokensOfSourceCode());
|
||||
|
||||
counter ++;
|
||||
if (counter % 100 == 0) {
|
||||
editScripts.setLength(0);
|
||||
patchesSourceCode.setLength(0);
|
||||
sizes.setLength(0);
|
||||
tokens.setLength(0);
|
||||
log.info("Worker #" + id +" finalized parsing " + counter + " files...");
|
||||
testingInfo.setLength(0);
|
||||
}
|
||||
counter ++;
|
||||
if (counter % 10 == 0) {
|
||||
log.info("Worker #" + id +" finalized parsing " + counter + " files... remaing "+ (files.size() - counter));
|
||||
}
|
||||
} catch (TimeoutException e) {
|
||||
future.cancel(true);
|
||||
@@ -116,17 +75,7 @@ public class EDiffWorker extends UntypedActor {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
if (sizes.length() > 0) {
|
||||
editScripts.setLength(0);
|
||||
patchesSourceCode.setLength(0);
|
||||
sizes.setLength(0);
|
||||
tokens.setLength(0);
|
||||
|
||||
testingInfo.setLength(0);
|
||||
}
|
||||
|
||||
// log.info("Worker #" + id +" finalized parsing " + counter + " files...");
|
||||
log.info("Worker #" + id + " finalized the work...");
|
||||
this.getSender().tell("STOP", getSelf());
|
||||
} else {
|
||||
|
||||
@@ -24,7 +24,7 @@ public class AkkaTreeLoader {
|
||||
private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class);
|
||||
|
||||
|
||||
public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, int cursor,String chunk,String eDiffTimeout) throws Exception {
|
||||
public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, int cursor, String chunk, String eDiffTimeout, String parallelism) throws Exception {
|
||||
|
||||
|
||||
String parameters = String.format("\nportInner %s \nchunkName %s \ndbDir %s \ndumpsName %s",portInner,chunkName,dbDir,dumpsName);
|
||||
@@ -90,17 +90,17 @@ public class AkkaTreeLoader {
|
||||
|
||||
|
||||
|
||||
akkaCompare(innerPool,outerPool,numOfWorkers,cursor,eDiffTimeout);
|
||||
akkaCompare(innerPool,outerPool,numOfWorkers,cursor,eDiffTimeout,parallelism);
|
||||
|
||||
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
|
||||
String stopServer2 = String.format(stopServer,Integer.valueOf(port));
|
||||
|
||||
cs.runShell(stopServer2,port);
|
||||
|
||||
|
||||
String stopServer1 = String.format(stopServer,Integer.valueOf(portInner));
|
||||
|
||||
cs.runShell(stopServer1,portInner);
|
||||
// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
|
||||
// String stopServer2 = String.format(stopServer,Integer.valueOf(port));
|
||||
//
|
||||
// cs.runShell(stopServer2,port);
|
||||
//
|
||||
//
|
||||
// String stopServer1 = String.format(stopServer,Integer.valueOf(portInner));
|
||||
//
|
||||
// cs.runShell(stopServer1,portInner);
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package edu.lu.uni.serval.fixminer.jobs;
|
||||
import akka.actor.ActorRef;
|
||||
import akka.actor.ActorSystem;
|
||||
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor;
|
||||
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser;
|
||||
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage;
|
||||
import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile;
|
||||
import edu.lu.uni.serval.utils.FileHelper;
|
||||
@@ -20,7 +21,7 @@ public class EnhancedASTDiff {
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class);
|
||||
|
||||
public static void main(String inputPath, String outputPath, String numOfWorkers, String project, String eDiffTimeout, String actionType) {
|
||||
public static void main(String inputPath, String outputPath, String numOfWorkers, String project, String eDiffTimeout, String actionType, String parallelism) {
|
||||
|
||||
|
||||
String parameters = String.format("\nInput path %s \nOutput path %s",inputPath,outputPath);
|
||||
@@ -34,43 +35,81 @@ public class EnhancedASTDiff {
|
||||
.collect(Collectors.toList());
|
||||
|
||||
|
||||
List<MessageFile> allMessageFiles = new ArrayList<>();
|
||||
for (File target : folders) {
|
||||
String pjName = target.getName();
|
||||
String pjName = target.getName();
|
||||
|
||||
|
||||
final List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
|
||||
System.out.println(msgFiles.size());
|
||||
if(msgFiles.size() == 0)
|
||||
continue;
|
||||
List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
|
||||
allMessageFiles.addAll(msgFiles);
|
||||
// System.out.println(msgFiles.size());
|
||||
if (msgFiles.size() == 0)
|
||||
continue;
|
||||
|
||||
String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/";
|
||||
String GUM_TREE_OUTPUT = outputPath + "/" + pjName + "/";
|
||||
|
||||
// a
|
||||
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD");
|
||||
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS");
|
||||
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL");
|
||||
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV");
|
||||
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/"+actionType);
|
||||
|
||||
|
||||
int a = 0;
|
||||
|
||||
ActorSystem system = null;
|
||||
ActorRef parsingActor = null;
|
||||
final EDiffMessage msg = new EDiffMessage(0, msgFiles,eDiffTimeout,actionType);
|
||||
try {
|
||||
log.info("Akka begins...");
|
||||
system = ActorSystem.create("Mining-FixPattern-System");
|
||||
|
||||
parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor");
|
||||
parsingActor.tell(msg, ActorRef.noSender());
|
||||
} catch (Exception e) {
|
||||
system.shutdown();
|
||||
e.printStackTrace();
|
||||
if (actionType.equals("ALL")) {
|
||||
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/" + actionType);
|
||||
} else {
|
||||
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD");
|
||||
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS");
|
||||
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL");
|
||||
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV");
|
||||
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MIX");
|
||||
}
|
||||
}
|
||||
|
||||
switch (parallelism){
|
||||
case "AKKA":
|
||||
ActorSystem system = null;
|
||||
ActorRef parsingActor = null;
|
||||
final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,actionType);
|
||||
try {
|
||||
log.info("Akka begins...");
|
||||
log.info("{} files to process ...", allMessageFiles.size());
|
||||
system = ActorSystem.create("Mining-FixPattern-System");
|
||||
|
||||
parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor");
|
||||
parsingActor.tell(msg, ActorRef.noSender());
|
||||
} catch (Exception e) {
|
||||
system.shutdown();
|
||||
e.printStackTrace();
|
||||
}
|
||||
break;
|
||||
case "FORKJOIN":
|
||||
int counter = new Object() {
|
||||
int counter = 0;
|
||||
|
||||
{
|
||||
allMessageFiles.stream().
|
||||
parallel().
|
||||
peek(x -> counter++).
|
||||
forEach(m ->
|
||||
{
|
||||
EDiffHunkParser parser = new EDiffHunkParser();
|
||||
parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,actionType);
|
||||
if (counter % 10 == 0) {
|
||||
log.info("Finalized parsing " + counter + " files... remaing " + (allMessageFiles.size() - counter));
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
}.counter;
|
||||
log.info("Finished parsing {} files",counter);
|
||||
break;
|
||||
default:
|
||||
log.error("Unknown parallelism {}", parallelism);
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
private static List<MessageFile> getMessageFiles(String gumTreeInput) {
|
||||
|
||||
@@ -17,18 +17,21 @@ import java.util.stream.Stream;
|
||||
public class ImportPairs2DB {
|
||||
private static Logger log = LoggerFactory.getLogger(ImportPairs2DB.class);
|
||||
|
||||
public static void main(String csvInputPath,String portInner,String dbDir,String datasetPath) throws Exception {
|
||||
public static void main(String csvInputPath, String portInner, String dbDir, String datasetPath, String chunk) throws Exception {
|
||||
|
||||
|
||||
String parameters = String.format("\nInput path %s \nportInner %s \ndbDir %s",csvInputPath,portInner,dbDir);
|
||||
log.info(parameters);
|
||||
|
||||
String[] splits = chunk.split("\\.");
|
||||
String chunkType = splits[splits.length-1];
|
||||
log.info("Chunk type {}",chunkType);
|
||||
|
||||
File folder = new File(csvInputPath);
|
||||
File[] subFolders = folder.listFiles();
|
||||
Stream<File> stream = Arrays.stream(subFolders);
|
||||
List<File> pjs = stream
|
||||
.filter(x -> x.getName().endsWith(".txt"))
|
||||
.filter(x -> x.getName().endsWith(chunkType))
|
||||
.collect(Collectors.toList());
|
||||
Integer portInt = Integer.valueOf(portInner);
|
||||
|
||||
|
||||
@@ -7,7 +7,10 @@ import com.github.gumtreediff.matchers.Matchers;
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
import com.github.gumtreediff.tree.TreeContext;
|
||||
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
|
||||
import edu.lu.uni.serval.utils.*;
|
||||
import edu.lu.uni.serval.utils.CallShell;
|
||||
import edu.lu.uni.serval.utils.EDiffHelper;
|
||||
import edu.lu.uni.serval.utils.FileHelper;
|
||||
import edu.lu.uni.serval.utils.PoolBuilder;
|
||||
import org.javatuples.Pair;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -145,7 +148,7 @@ public class MultiThreadTreeLoaderCluster {
|
||||
ITree parent = null;
|
||||
ITree children =null;
|
||||
TreeContext tc = new TreeContext();
|
||||
tree = EDiff.getActionTree(actionSet, parent, children,tc);
|
||||
tree = EDiffHelper.getActionTree(actionSet, parent, children,tc);
|
||||
tree.setParent(null);
|
||||
tc.validate();
|
||||
|
||||
|
||||
@@ -413,7 +413,7 @@ public class MultiThreadTreeLoaderCluster3 {
|
||||
}
|
||||
|
||||
// if (oldTokens.size() == 0 ) {// && (oldTree.getType() != 41 && oldTree.getType() != 21 && oldTree.getType() !=17 && oldTree.getType()!=60 && oldTree.getType() != 46)){
|
||||
// log.info("dur bakalim nereye!???");
|
||||
// log.info("!???");
|
||||
// }
|
||||
return oldTokens;
|
||||
}
|
||||
|
||||
@@ -2,19 +2,18 @@ package edu.lu.uni.serval.fixminer.jobs;
|
||||
|
||||
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
|
||||
import edu.lu.uni.serval.utils.Checker;
|
||||
import edu.lu.uni.serval.utils.EDiffHelper;
|
||||
import edu.lu.uni.serval.utils.PoolBuilder;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.Jedis;
|
||||
import redis.clients.jedis.JedisPool;
|
||||
import redis.clients.jedis.JedisPoolConfig;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
@@ -65,36 +64,9 @@ public class PatternExtractor {
|
||||
|
||||
|
||||
public static void loadDB(String inputPath,String portInner,String dbDir,String chunkName,String operation,List<String> fixes) throws Exception {
|
||||
// String inputPath;
|
||||
// String portInner;
|
||||
// String serverWait;
|
||||
// String dbDir;
|
||||
// String chunkName;
|
||||
// String numOfWorkers;
|
||||
// if (args.length > 0) {
|
||||
// inputPath = args[0];
|
||||
// portInner = args[1];
|
||||
// serverWait = args[2];
|
||||
// chunkName = args[3];
|
||||
// numOfWorkers = args[4];
|
||||
// dbDir = args[5];
|
||||
// } else {
|
||||
// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
|
||||
// portInner = "6399";
|
||||
// serverWait = "10000";
|
||||
// chunkName ="dumps.rdb";
|
||||
// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
|
||||
// numOfWorkers = "1";
|
||||
// }
|
||||
// String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s \noperation %s", inputPath, portInner, serverWait, chunkName, dbDir, operation);
|
||||
// log.info(parameters);
|
||||
// CallShell cs = new CallShell();
|
||||
// String cmd = "bash " + dbDir + "/" + "startServer.sh" + " %s %s %s";
|
||||
// cmd = String.format(cmd, dbDir, chunkName, Integer.valueOf(portInner));
|
||||
//// loadRedis(cmd,serverWait);
|
||||
// cs.runShell(cmd, serverWait);
|
||||
|
||||
JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(6399),20000000);
|
||||
|
||||
JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(6399),20000000);
|
||||
|
||||
|
||||
|
||||
@@ -134,11 +106,11 @@ public class PatternExtractor {
|
||||
s = inner.get(fn);
|
||||
if (s != null) {
|
||||
|
||||
HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s);
|
||||
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s);
|
||||
|
||||
|
||||
BufferedWriter writer = new BufferedWriter(new FileWriter(savePath+saveFN));
|
||||
writer.write(toString(actionSet));
|
||||
writer.write(EDiffHelper.toString(actionSet));
|
||||
|
||||
writer.close();
|
||||
|
||||
@@ -207,7 +179,7 @@ public class PatternExtractor {
|
||||
String saveFN = pattern.getName() + "_" + project + "_" + operation + "_" + joinFN;
|
||||
try{
|
||||
String content = new String(Files.readAllBytes(Paths.get(savePath + saveFN)));
|
||||
HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(content);
|
||||
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(content);
|
||||
|
||||
int astType = actionSet.getNode().getType();
|
||||
if (Checker.isStatement(astType) || astType == 23 //FieldDeclaration
|
||||
@@ -264,41 +236,7 @@ public class PatternExtractor {
|
||||
|
||||
|
||||
|
||||
static final JedisPoolConfig poolConfig = buildPoolConfig();
|
||||
|
||||
private static String toString( Serializable o ) throws IOException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
ObjectOutputStream oos = new ObjectOutputStream( baos );
|
||||
oos.writeObject( o );
|
||||
oos.close();
|
||||
return Base64.getEncoder().encodeToString(baos.toByteArray());
|
||||
}
|
||||
|
||||
/** Read the object from Base64 string. */
|
||||
private static Object fromString( String s ) throws IOException ,
|
||||
ClassNotFoundException {
|
||||
byte[] data = Base64.getDecoder().decode(s);
|
||||
ObjectInputStream ois = new ObjectInputStream(
|
||||
new ByteArrayInputStream(data));
|
||||
Object o = ois.readObject();
|
||||
ois.close();
|
||||
return o;
|
||||
}
|
||||
private static JedisPoolConfig buildPoolConfig() {
|
||||
final JedisPoolConfig poolConfig = new JedisPoolConfig();
|
||||
poolConfig.setMaxTotal(128);
|
||||
poolConfig.setMaxIdle(128);
|
||||
poolConfig.setMinIdle(16);
|
||||
poolConfig.setTestOnBorrow(false);
|
||||
poolConfig.setTestOnReturn(false);
|
||||
poolConfig.setTestWhileIdle(true);
|
||||
poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis());
|
||||
poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis());
|
||||
// poolConfig.setNumTestsPerEvictionRun(3);
|
||||
poolConfig.setBlockWhenExhausted(true);
|
||||
|
||||
return poolConfig;
|
||||
}
|
||||
|
||||
|
||||
public static List<String> readIdList(String fileName) {
|
||||
|
||||
@@ -48,24 +48,24 @@ public class StoreEDiffInCache {
|
||||
File[] files = pj.listFiles();
|
||||
Stream<File> fileStream = Arrays.stream(files);
|
||||
List<File> fs;
|
||||
if (operation.equals("ALLOP")){
|
||||
fs= fileStream
|
||||
.filter(x -> x.getName().startsWith("UPD") ||
|
||||
x.getName().startsWith("INS") ||
|
||||
x.getName().startsWith("DEL") ||
|
||||
x.getName().startsWith("MOV"))
|
||||
.collect(Collectors.toList());
|
||||
File[] files1 = fs.get(0).listFiles();
|
||||
File[] files2 = fs.get(1).listFiles();
|
||||
File[] files3 = fs.get(2).listFiles();
|
||||
File[] files4 = fs.get(3).listFiles();
|
||||
dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new);
|
||||
}else{
|
||||
// if (operation.equals("ALLOP")){
|
||||
// fs= fileStream
|
||||
// .filter(x -> x.getName().startsWith("UPD") ||
|
||||
// x.getName().startsWith("INS") ||
|
||||
// x.getName().startsWith("DEL") ||
|
||||
// x.getName().startsWith("MOV"))
|
||||
// .collect(Collectors.toList());
|
||||
// File[] files1 = fs.get(0).listFiles();
|
||||
// File[] files2 = fs.get(1).listFiles();
|
||||
// File[] files3 = fs.get(2).listFiles();
|
||||
// File[] files4 = fs.get(3).listFiles();
|
||||
// dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new);
|
||||
// }else{
|
||||
fs = fileStream
|
||||
.filter(x -> x.getName().startsWith(operation))
|
||||
.collect(Collectors.toList());
|
||||
dumps = fs.get(0).listFiles();
|
||||
}
|
||||
// }
|
||||
|
||||
|
||||
for (File f : dumps) {
|
||||
@@ -115,7 +115,7 @@ public class StoreEDiffInCache {
|
||||
log.error("File not found {}" , path);
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
log.error("Error initializing stream {}" , path);
|
||||
log.error("Error initializing stream {}", path);
|
||||
e.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
// TODO Auto-generated catch block
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
package edu.lu.uni.serval.utils;
|
||||
|
||||
import com.github.gumtreediff.actions.model.*;
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
import com.github.gumtreediff.tree.TreeContext;
|
||||
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.Jedis;
|
||||
import redis.clients.jedis.JedisPool;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Created by anilkoyuncu on 17/09/2018.
|
||||
*/
|
||||
public class EDiff {
|
||||
private static Logger log = LoggerFactory.getLogger(EDiff.class);
|
||||
public static ITree getSimpliedTree(String fn, JedisPool outerPool) {
|
||||
|
||||
ITree tree = null;
|
||||
Jedis inner = null;
|
||||
try {
|
||||
inner = outerPool.getResource();
|
||||
while (!inner.ping().equals("PONG")){
|
||||
log.info("wait");
|
||||
}
|
||||
inner.select(1);
|
||||
String dist2load = inner.get(fn);
|
||||
inner.select(0);
|
||||
String s = inner.get(dist2load);
|
||||
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s);
|
||||
|
||||
ITree parent = null;
|
||||
ITree children =null;
|
||||
TreeContext tc = new TreeContext();
|
||||
tree = EDiffHelper.getASTTree(actionSet, parent, children,tc);
|
||||
tree.setParent(null);
|
||||
tc.validate();
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}finally {
|
||||
if (inner != null) {
|
||||
inner.close();
|
||||
}
|
||||
}
|
||||
return tree;
|
||||
|
||||
}
|
||||
public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){
|
||||
|
||||
int newType = 0;
|
||||
|
||||
Action action = actionSet.getAction();
|
||||
if (action instanceof Update){
|
||||
newType = 101;
|
||||
}else if(action instanceof Insert){
|
||||
newType =100;
|
||||
}else if(action instanceof Move){
|
||||
newType = 102;
|
||||
}else if(action instanceof Delete){
|
||||
newType=103;
|
||||
}else{
|
||||
new Exception("unknow action");
|
||||
}
|
||||
if(actionSet.getParent() == null){
|
||||
//root
|
||||
|
||||
parent = tc.createTree(newType, "", null);
|
||||
tc.setRoot(parent);
|
||||
|
||||
// parent = new Tree(newType,"");
|
||||
}else{
|
||||
children = tc.createTree(newType, "", null);
|
||||
children.setParentAndUpdateChildren(parent);
|
||||
// children = new Tree(newType,"");
|
||||
// parent.addChild(children);
|
||||
}
|
||||
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
|
||||
if (subActions.size() != 0){
|
||||
for (HierarchicalActionSet subAction : subActions) {
|
||||
|
||||
if(actionSet.getParent() == null){
|
||||
children = parent;
|
||||
}
|
||||
getActionTree(subAction,children,null,tc);
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
return parent;
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,13 @@
|
||||
package edu.lu.uni.serval.utils;
|
||||
|
||||
import com.github.gumtreediff.actions.model.*;
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
import com.github.gumtreediff.tree.TreeContext;
|
||||
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.Jedis;
|
||||
import redis.clients.jedis.JedisPool;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.Base64;
|
||||
@@ -126,4 +129,81 @@ public class EDiffHelper {
|
||||
.map(Map.Entry::getKey)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public static ITree getSimpliedTree(String fn, JedisPool outerPool) {
|
||||
|
||||
ITree tree = null;
|
||||
Jedis inner = null;
|
||||
try {
|
||||
inner = outerPool.getResource();
|
||||
while (!inner.ping().equals("PONG")){
|
||||
log.info("wait");
|
||||
}
|
||||
inner.select(1);
|
||||
String dist2load = inner.get(fn);
|
||||
inner.select(0);
|
||||
String s = inner.get(dist2load);
|
||||
HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s);
|
||||
|
||||
ITree parent = null;
|
||||
ITree children =null;
|
||||
TreeContext tc = new TreeContext();
|
||||
tree = EDiffHelper.getASTTree(actionSet, parent, children,tc);
|
||||
tree.setParent(null);
|
||||
tc.validate();
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}finally {
|
||||
if (inner != null) {
|
||||
inner.close();
|
||||
}
|
||||
}
|
||||
return tree;
|
||||
|
||||
}
|
||||
public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){
|
||||
|
||||
int newType = 0;
|
||||
|
||||
Action action = actionSet.getAction();
|
||||
if (action instanceof Update){
|
||||
newType = 101;
|
||||
}else if(action instanceof Insert){
|
||||
newType =100;
|
||||
}else if(action instanceof Move){
|
||||
newType = 102;
|
||||
}else if(action instanceof Delete){
|
||||
newType=103;
|
||||
}else{
|
||||
new Exception("unknow action");
|
||||
}
|
||||
if(actionSet.getParent() == null){
|
||||
//root
|
||||
|
||||
parent = tc.createTree(newType, "", null);
|
||||
tc.setRoot(parent);
|
||||
|
||||
}else{
|
||||
children = tc.createTree(newType, "", null);
|
||||
children.setParentAndUpdateChildren(parent);
|
||||
|
||||
}
|
||||
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
|
||||
if (subActions.size() != 0){
|
||||
for (HierarchicalActionSet subAction : subActions) {
|
||||
|
||||
if(actionSet.getParent() == null){
|
||||
children = parent;
|
||||
}
|
||||
getActionTree(subAction,children,null,tc);
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
return parent;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user