azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

red

Browse Source
This commit is contained in:
Anil Koyuncu
2018-03-24 18:55:00 +01:00
parent 3f82ebd57d
commit 884016e612
1 changed files with 155 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java
+155 -84
View File
@@ -20,6 +20,8 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.Deflater;
@@ -30,6 +32,23 @@ import java.util.zip.GZIPOutputStream;
* Created by anilkoyuncu on 19/03/2018.
*/
public class MultiThreadTreeLoader {
private static class StreamGobbler implements Runnable {
private InputStream inputStream;
private Consumer<String> consumer;
public StreamGobbler(InputStream inputStream, Consumer<String> consumer) {
this.inputStream = inputStream;
this.consumer = consumer;
}
@Override
public void run() {
new BufferedReader(new InputStreamReader(inputStream)).lines()
.forEach(consumer);
}
}
private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoader.class);
public static void main(String[] args) {
@@ -37,56 +56,107 @@ public class MultiThreadTreeLoader {
String inputPath;
String outputPath;
String port;
String pairsCSVPath;
String importScript;
if (args.length > 0) {
inputPath = args[0];
outputPath = args[1];
port = args[2];
pairsCSVPath = args[3];
importScript = args[4];
} else {
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2/";
outputPath = "/Users/anilkoyuncu/bugStudy/dataset/";
port = "6379";
pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test";
importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh";
}
String cmd;
cmd = "bash " + importScript +" %s";
JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
// calculatePairs(inputPath, outputPath);
// processMessages(inputPath,outputPath);
// evaluateResults(inputPath,outputPath);
try (Jedis jedis = jedisPool.getResource()) {
// do operations with jedis resource
Set<String> names = jedis.keys("*");
ScanParams sc = new ScanParams();
sc.count(150000000);
sc.match("pair_*");
ScanResult<String> scan = jedis.scan("0",sc);
File folder = new File(pairsCSVPath);
File[] listOfFiles = folder.listFiles();
Stream<File> stream = Arrays.stream(listOfFiles);
List<File> folders = stream
.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
for (File f:folders){
Process process;
try {
String comd = String.format(cmd, f.getAbsoluteFile());
process = Runtime.getRuntime()
.exec(comd);
StreamGobbler streamGobbler =
new StreamGobbler(process.getInputStream(), System.out::println);
Executors.newSingleThreadExecutor().submit(streamGobbler);
int exitCode = process.waitFor();
assert exitCode == 0;
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
try (Jedis jedis = jedisPool.getResource()) {
// do operations with jedis resource
// Set<String> names = jedis.keys("*");
ScanParams sc = new ScanParams();
sc.count(150000000);
sc.match("pair_*");
jedis.configSet("dbfilename",f.getName());
ScanResult<String> scan = jedis.scan("0",sc);
// java.util.Iterator<String> it = names.iterator();
// while(it.hasNext()) {
// String s = it.next();
// System.out.println(s + " : " + jedis.get(s));
// }
scan.getResult().parallelStream()
.forEach(m -> coreCompare(m, inputPath, jedisPool));
jedis.save();
}
// java.util.Iterator<String> it = names.iterator();
// while(it.hasNext()) {
// String s = it.next();
// System.out.println(s + " : " + jedis.get(s));
// }
scan.getResult().parallelStream()
.forEach(m -> coreCompare(m, inputPath));
}
}
private static void coreCompare(String name , String inputPath) {
private static void coreCompare(String name , String inputPath, JedisPool jedisPool) {
String value;
try (Jedis jedis = jedisPool.getResource()) {
value = jedis.get(name);
Map<String, String> resultMap = jedis.hgetAll(name);
resultMap.get("0");
String[] split = value.split(",");
String[] split = name.split("_");
log.info("Starting in coreLoop");
String i = split[0];
String j = split[1];
String firstValue = split[2];
String secondValue = split[3];
String i = split[1];
String j = split[2];
String firstValue = resultMap.get("0");
String secondValue = resultMap.get("1");
String[] firstValueSplit = firstValue.split("GumTreeOutput2");
String[] secondValueSplit = secondValue.split("GumTreeOutput2");
@@ -124,13 +194,14 @@ public class MultiThreadTreeLoader {
String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1);
String editDistance = String.valueOf(actions.size());
jedis.select(1);
String result = chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
jedis.set(resultKey, result);
if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
|| ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0){
String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j);
jedis.select(2);
jedis.set(matchKey, result);
}
@@ -176,17 +247,17 @@ public class MultiThreadTreeLoader {
.forEach(m -> coreLoop(m, outputPath,inputPath));
}
public static void evaluateResults(String inputPath, String outputPath){
File folder = new File(outputPath + "comparison_splitted/");
File[] listOfFiles = folder.listFiles();
Stream<File> stream = Arrays.stream(listOfFiles);
List<File> pjs = stream
.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
FileHelper.createDirectory(outputPath + "eval_splitted/");
pjs.parallelStream()
.forEach(m -> coreEval(m, outputPath,inputPath));
}
// public static void evaluateResults(String inputPath, String outputPath){
// File folder = new File(outputPath + "comparison_splitted/");
// File[] listOfFiles = folder.listFiles();
// Stream<File> stream = Arrays.stream(listOfFiles);
// List<File> pjs = stream
// .filter(x -> !x.getName().startsWith("."))
// .collect(Collectors.toList());
// FileHelper.createDirectory(outputPath + "eval_splitted/");
// pjs.parallelStream()
// .forEach(m -> coreEval(m, outputPath,inputPath));
// }
public static ITree getSimpliedTree(String fn) {
ITree tree = null;
@@ -220,55 +291,55 @@ public class MultiThreadTreeLoader {
}
private static void coreEval(File mes, String outputPath,String inputPath) {
try {
log.info("Starting in coreLoop");
BufferedReader br = null;
String sCurrentLine = null;
try (Jedis jedis = jedisPool.getResource()) {
// do operations with jedis resource
Set<String> names = jedis.keys("*");
java.util.Iterator<String> it = names.iterator();
while(it.hasNext()) {
String s = it.next();
System.out.println(s + " : " + jedis.get(s));
}
}
BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "eval_splitted/" + "eval_" + mes.getName()));
br = new BufferedReader(
new FileReader(mes));
while ((sCurrentLine = br.readLine()) != null) {
String currentLine = sCurrentLine;
String[] split = currentLine.split("\t");
String i = split[0];
String j = split[1];
Double chawatheSimilarity = Double.valueOf(split[2]);
Double diceSimilarity = Double.valueOf(split[3]);
Double jaccardSimilarity = Double.valueOf(split[4]);
int actionSize = Integer.valueOf(split[5]);
// String firstValue = split[6];
// String secondValue = split[7];
if (chawatheSimilarity.equals(1.0) || diceSimilarity.equals(1.0) || jaccardSimilarity.equals(1.0) || actionSize == 0){
writer.write(currentLine);
writer.write("\n");
}
}
writer.close();
} catch (FileNotFoundException e) {
log.error("File not found");
e.printStackTrace();
} catch (IOException e) {
log.error("Error initializing stream");
e.printStackTrace();
}
}
// private static void coreEval(File mes, String outputPath,String inputPath) {
// try {
//
// log.info("Starting in coreLoop");
//
// BufferedReader br = null;
// String sCurrentLine = null;
//
// try (Jedis jedis = jedisPool.getResource()) {
// // do operations with jedis resource
// Set<String> names = jedis.keys("*");
//
// java.util.Iterator<String> it = names.iterator();
// while(it.hasNext()) {
// String s = it.next();
// System.out.println(s + " : " + jedis.get(s));
// }
// }
//
// BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "eval_splitted/" + "eval_" + mes.getName()));
//
// br = new BufferedReader(
// new FileReader(mes));
// while ((sCurrentLine = br.readLine()) != null) {
// String currentLine = sCurrentLine;
// String[] split = currentLine.split("\t");
// String i = split[0];
// String j = split[1];
// Double chawatheSimilarity = Double.valueOf(split[2]);
// Double diceSimilarity = Double.valueOf(split[3]);
// Double jaccardSimilarity = Double.valueOf(split[4]);
// int actionSize = Integer.valueOf(split[5]);
//// String firstValue = split[6];
//// String secondValue = split[7];
// if (chawatheSimilarity.equals(1.0) || diceSimilarity.equals(1.0) || jaccardSimilarity.equals(1.0) || actionSize == 0){
// writer.write(currentLine);
// writer.write("\n");
// }
// }
// writer.close();
// } catch (FileNotFoundException e) {
// log.error("File not found");
// e.printStackTrace();
// } catch (IOException e) {
// log.error("Error initializing stream");
// e.printStackTrace();
//
// }
// }
private static void coreLoop(File mes, String outputPath,String inputPath) {
try {
@@ -391,7 +462,7 @@ public class MultiThreadTreeLoader {
}
static final JedisPoolConfig poolConfig = buildPoolConfig();
static JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",6379,20000000);
private static JedisPoolConfig buildPoolConfig() {
final JedisPoolConfig poolConfig = new JedisPoolConfig();