azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

red

Browse Source
This commit is contained in:
Anil Koyuncu
2018-03-24 18:55:00 +01:00
parent 3f82ebd57d
commit 884016e612
1 changed files with 155 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java
+155 -84
View File
@@ -20,6 +20,8 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel; import java.nio.channels.FileChannel;
import java.time.Duration; import java.time.Duration;
import java.util.*; import java.util.*;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.zip.Deflater; import java.util.zip.Deflater;
@@ -30,6 +32,23 @@ import java.util.zip.GZIPOutputStream;
* Created by anilkoyuncu on 19/03/2018. * Created by anilkoyuncu on 19/03/2018.
*/ */
public class MultiThreadTreeLoader { public class MultiThreadTreeLoader {
private static class StreamGobbler implements Runnable {
private InputStream inputStream;
private Consumer<String> consumer;
public StreamGobbler(InputStream inputStream, Consumer<String> consumer) {
this.inputStream = inputStream;
this.consumer = consumer;
}
@Override
public void run() {
new BufferedReader(new InputStreamReader(inputStream)).lines()
.forEach(consumer);
}
}
private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoader.class); private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoader.class);
public static void main(String[] args) { public static void main(String[] args) {
@@ -37,56 +56,107 @@ public class MultiThreadTreeLoader {
String inputPath; String inputPath;
String outputPath; String outputPath;
String port;
String pairsCSVPath;
String importScript;
if (args.length > 0) { if (args.length > 0) {
inputPath = args[0]; inputPath = args[0];
outputPath = args[1]; outputPath = args[1];
port = args[2];
pairsCSVPath = args[3];
importScript = args[4];
} else { } else {
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2/"; inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2/";
outputPath = "/Users/anilkoyuncu/bugStudy/dataset/"; outputPath = "/Users/anilkoyuncu/bugStudy/dataset/";
port = "6379";
pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test";
importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh";
} }
String cmd;
cmd = "bash " + importScript +" %s";
JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
// calculatePairs(inputPath, outputPath); // calculatePairs(inputPath, outputPath);
// processMessages(inputPath,outputPath); // processMessages(inputPath,outputPath);
// evaluateResults(inputPath,outputPath); // evaluateResults(inputPath,outputPath);
try (Jedis jedis = jedisPool.getResource()) { File folder = new File(pairsCSVPath);
// do operations with jedis resource File[] listOfFiles = folder.listFiles();
Set<String> names = jedis.keys("*"); Stream<File> stream = Arrays.stream(listOfFiles);
ScanParams sc = new ScanParams(); List<File> folders = stream
sc.count(150000000); .filter(x -> !x.getName().startsWith("."))
sc.match("pair_*"); .collect(Collectors.toList());
ScanResult<String> scan = jedis.scan("0",sc);
for (File f:folders){
Process process;
try {
String comd = String.format(cmd, f.getAbsoluteFile());
process = Runtime.getRuntime()
.exec(comd);
StreamGobbler streamGobbler =
new StreamGobbler(process.getInputStream(), System.out::println);
Executors.newSingleThreadExecutor().submit(streamGobbler);
int exitCode = process.waitFor();
assert exitCode == 0;
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
try (Jedis jedis = jedisPool.getResource()) {
// do operations with jedis resource
// Set<String> names = jedis.keys("*");
ScanParams sc = new ScanParams();
sc.count(150000000);
sc.match("pair_*");
jedis.configSet("dbfilename",f.getName());
ScanResult<String> scan = jedis.scan("0",sc);
// java.util.Iterator<String> it = names.iterator();
// while(it.hasNext()) {
// String s = it.next();
// System.out.println(s + " : " + jedis.get(s));
// }
scan.getResult().parallelStream()
.forEach(m -> coreCompare(m, inputPath, jedisPool));
jedis.save();
}
// java.util.Iterator<String> it = names.iterator();
// while(it.hasNext()) {
// String s = it.next();
// System.out.println(s + " : " + jedis.get(s));
// }
scan.getResult().parallelStream()
.forEach(m -> coreCompare(m, inputPath));
} }
} }
private static void coreCompare(String name , String inputPath) { private static void coreCompare(String name , String inputPath, JedisPool jedisPool) {
String value;
try (Jedis jedis = jedisPool.getResource()) { try (Jedis jedis = jedisPool.getResource()) {
value = jedis.get(name); Map<String, String> resultMap = jedis.hgetAll(name);
resultMap.get("0");
String[] split = value.split(","); String[] split = name.split("_");
log.info("Starting in coreLoop"); log.info("Starting in coreLoop");
String i = split[0]; String i = split[1];
String j = split[1]; String j = split[2];
String firstValue = split[2]; String firstValue = resultMap.get("0");
String secondValue = split[3]; String secondValue = resultMap.get("1");
String[] firstValueSplit = firstValue.split("GumTreeOutput2"); String[] firstValueSplit = firstValue.split("GumTreeOutput2");
String[] secondValueSplit = secondValue.split("GumTreeOutput2"); String[] secondValueSplit = secondValue.split("GumTreeOutput2");
@@ -124,13 +194,14 @@ public class MultiThreadTreeLoader {
String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1);
String editDistance = String.valueOf(actions.size()); String editDistance = String.valueOf(actions.size());
jedis.select(1);
String result = chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; String result = chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
jedis.set(resultKey, result); jedis.set(resultKey, result);
if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
|| ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0){ || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0){
String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j);
jedis.select(2);
jedis.set(matchKey, result); jedis.set(matchKey, result);
} }
@@ -176,17 +247,17 @@ public class MultiThreadTreeLoader {
.forEach(m -> coreLoop(m, outputPath,inputPath)); .forEach(m -> coreLoop(m, outputPath,inputPath));
} }
public static void evaluateResults(String inputPath, String outputPath){ // public static void evaluateResults(String inputPath, String outputPath){
File folder = new File(outputPath + "comparison_splitted/"); // File folder = new File(outputPath + "comparison_splitted/");
File[] listOfFiles = folder.listFiles(); // File[] listOfFiles = folder.listFiles();
Stream<File> stream = Arrays.stream(listOfFiles); // Stream<File> stream = Arrays.stream(listOfFiles);
List<File> pjs = stream // List<File> pjs = stream
.filter(x -> !x.getName().startsWith(".")) // .filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList()); // .collect(Collectors.toList());
FileHelper.createDirectory(outputPath + "eval_splitted/"); // FileHelper.createDirectory(outputPath + "eval_splitted/");
pjs.parallelStream() // pjs.parallelStream()
.forEach(m -> coreEval(m, outputPath,inputPath)); // .forEach(m -> coreEval(m, outputPath,inputPath));
} // }
public static ITree getSimpliedTree(String fn) { public static ITree getSimpliedTree(String fn) {
ITree tree = null; ITree tree = null;
@@ -220,55 +291,55 @@ public class MultiThreadTreeLoader {
} }
private static void coreEval(File mes, String outputPath,String inputPath) { // private static void coreEval(File mes, String outputPath,String inputPath) {
try { // try {
//
log.info("Starting in coreLoop"); // log.info("Starting in coreLoop");
//
BufferedReader br = null; // BufferedReader br = null;
String sCurrentLine = null; // String sCurrentLine = null;
//
try (Jedis jedis = jedisPool.getResource()) { // try (Jedis jedis = jedisPool.getResource()) {
// do operations with jedis resource // // do operations with jedis resource
Set<String> names = jedis.keys("*"); // Set<String> names = jedis.keys("*");
//
java.util.Iterator<String> it = names.iterator(); // java.util.Iterator<String> it = names.iterator();
while(it.hasNext()) { // while(it.hasNext()) {
String s = it.next(); // String s = it.next();
System.out.println(s + " : " + jedis.get(s)); // System.out.println(s + " : " + jedis.get(s));
} // }
} // }
//
BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "eval_splitted/" + "eval_" + mes.getName())); // BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "eval_splitted/" + "eval_" + mes.getName()));
//
br = new BufferedReader( // br = new BufferedReader(
new FileReader(mes)); // new FileReader(mes));
while ((sCurrentLine = br.readLine()) != null) { // while ((sCurrentLine = br.readLine()) != null) {
String currentLine = sCurrentLine; // String currentLine = sCurrentLine;
String[] split = currentLine.split("\t"); // String[] split = currentLine.split("\t");
String i = split[0]; // String i = split[0];
String j = split[1]; // String j = split[1];
Double chawatheSimilarity = Double.valueOf(split[2]); // Double chawatheSimilarity = Double.valueOf(split[2]);
Double diceSimilarity = Double.valueOf(split[3]); // Double diceSimilarity = Double.valueOf(split[3]);
Double jaccardSimilarity = Double.valueOf(split[4]); // Double jaccardSimilarity = Double.valueOf(split[4]);
int actionSize = Integer.valueOf(split[5]); // int actionSize = Integer.valueOf(split[5]);
// String firstValue = split[6]; //// String firstValue = split[6];
// String secondValue = split[7]; //// String secondValue = split[7];
if (chawatheSimilarity.equals(1.0) || diceSimilarity.equals(1.0) || jaccardSimilarity.equals(1.0) || actionSize == 0){ // if (chawatheSimilarity.equals(1.0) || diceSimilarity.equals(1.0) || jaccardSimilarity.equals(1.0) || actionSize == 0){
writer.write(currentLine); // writer.write(currentLine);
writer.write("\n"); // writer.write("\n");
} // }
} // }
writer.close(); // writer.close();
} catch (FileNotFoundException e) { // } catch (FileNotFoundException e) {
log.error("File not found"); // log.error("File not found");
e.printStackTrace(); // e.printStackTrace();
} catch (IOException e) { // } catch (IOException e) {
log.error("Error initializing stream"); // log.error("Error initializing stream");
e.printStackTrace(); // e.printStackTrace();
//
} // }
} // }
private static void coreLoop(File mes, String outputPath,String inputPath) { private static void coreLoop(File mes, String outputPath,String inputPath) {
try { try {
@@ -391,7 +462,7 @@ public class MultiThreadTreeLoader {
} }
static final JedisPoolConfig poolConfig = buildPoolConfig(); static final JedisPoolConfig poolConfig = buildPoolConfig();
static JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",6379,20000000);
private static JedisPoolConfig buildPoolConfig() { private static JedisPoolConfig buildPoolConfig() {
final JedisPoolConfig poolConfig = new JedisPoolConfig(); final JedisPoolConfig poolConfig = new JedisPoolConfig();