diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java index 5e577b0..b7b6f76 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java @@ -20,6 +20,8 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.time.Duration; import java.util.*; +import java.util.concurrent.Executors; +import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.Deflater; @@ -30,6 +32,23 @@ import java.util.zip.GZIPOutputStream; * Created by anilkoyuncu on 19/03/2018. */ public class MultiThreadTreeLoader { + + private static class StreamGobbler implements Runnable { + private InputStream inputStream; + private Consumer consumer; + + public StreamGobbler(InputStream inputStream, Consumer consumer) { + this.inputStream = inputStream; + this.consumer = consumer; + } + + @Override + public void run() { + new BufferedReader(new InputStreamReader(inputStream)).lines() + .forEach(consumer); + } + } + private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoader.class); public static void main(String[] args) { @@ -37,56 +56,107 @@ public class MultiThreadTreeLoader { String inputPath; String outputPath; + String port; + String pairsCSVPath; + String importScript; if (args.length > 0) { inputPath = args[0]; outputPath = args[1]; + port = args[2]; + pairsCSVPath = args[3]; + importScript = args[4]; } else { inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2/"; outputPath = "/Users/anilkoyuncu/bugStudy/dataset/"; + port = "6379"; + pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test"; + importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh"; } + String cmd; + cmd = "bash " + importScript +" %s"; + JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); // calculatePairs(inputPath, outputPath); // processMessages(inputPath,outputPath); // evaluateResults(inputPath,outputPath); - try (Jedis jedis = jedisPool.getResource()) { - // do operations with jedis resource - Set names = jedis.keys("*"); - ScanParams sc = new ScanParams(); - sc.count(150000000); - sc.match("pair_*"); - ScanResult scan = jedis.scan("0",sc); + File folder = new File(pairsCSVPath); + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List folders = stream + .filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); + + for (File f:folders){ + + + Process process; + + try { + String comd = String.format(cmd, f.getAbsoluteFile()); + process = Runtime.getRuntime() + + .exec(comd); + + + StreamGobbler streamGobbler = + new StreamGobbler(process.getInputStream(), System.out::println); + Executors.newSingleThreadExecutor().submit(streamGobbler); + int exitCode = process.waitFor(); + assert exitCode == 0; + } catch (IOException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + + try (Jedis jedis = jedisPool.getResource()) { + // do operations with jedis resource + // Set names = jedis.keys("*"); + ScanParams sc = new ScanParams(); + sc.count(150000000); + sc.match("pair_*"); + jedis.configSet("dbfilename",f.getName()); + + ScanResult scan = jedis.scan("0",sc); + + // java.util.Iterator it = names.iterator(); + // while(it.hasNext()) { + // String s = it.next(); + // System.out.println(s + " : " + jedis.get(s)); + // } + scan.getResult().parallelStream() + .forEach(m -> coreCompare(m, inputPath, jedisPool)); + + jedis.save(); + } -// java.util.Iterator it = names.iterator(); -// while(it.hasNext()) { -// String s = it.next(); -// System.out.println(s + " : " + jedis.get(s)); -// } - scan.getResult().parallelStream() - .forEach(m -> coreCompare(m, inputPath)); } } - private static void coreCompare(String name , String inputPath) { + private static void coreCompare(String name , String inputPath, JedisPool jedisPool) { + - String value; try (Jedis jedis = jedisPool.getResource()) { - value = jedis.get(name); + Map resultMap = jedis.hgetAll(name); + + resultMap.get("0"); - String[] split = value.split(","); + String[] split = name.split("_"); log.info("Starting in coreLoop"); - String i = split[0]; - String j = split[1]; - String firstValue = split[2]; - String secondValue = split[3]; + String i = split[1]; + String j = split[2]; + String firstValue = resultMap.get("0"); + String secondValue = resultMap.get("1"); String[] firstValueSplit = firstValue.split("GumTreeOutput2"); String[] secondValueSplit = secondValue.split("GumTreeOutput2"); @@ -124,13 +194,14 @@ public class MultiThreadTreeLoader { String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); String editDistance = String.valueOf(actions.size()); - + jedis.select(1); String result = chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; jedis.set(resultKey, result); if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0){ String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); + jedis.select(2); jedis.set(matchKey, result); } @@ -176,17 +247,17 @@ public class MultiThreadTreeLoader { .forEach(m -> coreLoop(m, outputPath,inputPath)); } - public static void evaluateResults(String inputPath, String outputPath){ - File folder = new File(outputPath + "comparison_splitted/"); - File[] listOfFiles = folder.listFiles(); - Stream stream = Arrays.stream(listOfFiles); - List pjs = stream - .filter(x -> !x.getName().startsWith(".")) - .collect(Collectors.toList()); - FileHelper.createDirectory(outputPath + "eval_splitted/"); - pjs.parallelStream() - .forEach(m -> coreEval(m, outputPath,inputPath)); - } +// public static void evaluateResults(String inputPath, String outputPath){ +// File folder = new File(outputPath + "comparison_splitted/"); +// File[] listOfFiles = folder.listFiles(); +// Stream stream = Arrays.stream(listOfFiles); +// List pjs = stream +// .filter(x -> !x.getName().startsWith(".")) +// .collect(Collectors.toList()); +// FileHelper.createDirectory(outputPath + "eval_splitted/"); +// pjs.parallelStream() +// .forEach(m -> coreEval(m, outputPath,inputPath)); +// } public static ITree getSimpliedTree(String fn) { ITree tree = null; @@ -220,55 +291,55 @@ public class MultiThreadTreeLoader { } - private static void coreEval(File mes, String outputPath,String inputPath) { - try { - - log.info("Starting in coreLoop"); - - BufferedReader br = null; - String sCurrentLine = null; - - try (Jedis jedis = jedisPool.getResource()) { - // do operations with jedis resource - Set names = jedis.keys("*"); - - java.util.Iterator it = names.iterator(); - while(it.hasNext()) { - String s = it.next(); - System.out.println(s + " : " + jedis.get(s)); - } - } - - BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "eval_splitted/" + "eval_" + mes.getName())); - - br = new BufferedReader( - new FileReader(mes)); - while ((sCurrentLine = br.readLine()) != null) { - String currentLine = sCurrentLine; - String[] split = currentLine.split("\t"); - String i = split[0]; - String j = split[1]; - Double chawatheSimilarity = Double.valueOf(split[2]); - Double diceSimilarity = Double.valueOf(split[3]); - Double jaccardSimilarity = Double.valueOf(split[4]); - int actionSize = Integer.valueOf(split[5]); -// String firstValue = split[6]; -// String secondValue = split[7]; - if (chawatheSimilarity.equals(1.0) || diceSimilarity.equals(1.0) || jaccardSimilarity.equals(1.0) || actionSize == 0){ - writer.write(currentLine); - writer.write("\n"); - } - } - writer.close(); - } catch (FileNotFoundException e) { - log.error("File not found"); - e.printStackTrace(); - } catch (IOException e) { - log.error("Error initializing stream"); - e.printStackTrace(); - - } - } +// private static void coreEval(File mes, String outputPath,String inputPath) { +// try { +// +// log.info("Starting in coreLoop"); +// +// BufferedReader br = null; +// String sCurrentLine = null; +// +// try (Jedis jedis = jedisPool.getResource()) { +// // do operations with jedis resource +// Set names = jedis.keys("*"); +// +// java.util.Iterator it = names.iterator(); +// while(it.hasNext()) { +// String s = it.next(); +// System.out.println(s + " : " + jedis.get(s)); +// } +// } +// +// BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "eval_splitted/" + "eval_" + mes.getName())); +// +// br = new BufferedReader( +// new FileReader(mes)); +// while ((sCurrentLine = br.readLine()) != null) { +// String currentLine = sCurrentLine; +// String[] split = currentLine.split("\t"); +// String i = split[0]; +// String j = split[1]; +// Double chawatheSimilarity = Double.valueOf(split[2]); +// Double diceSimilarity = Double.valueOf(split[3]); +// Double jaccardSimilarity = Double.valueOf(split[4]); +// int actionSize = Integer.valueOf(split[5]); +//// String firstValue = split[6]; +//// String secondValue = split[7]; +// if (chawatheSimilarity.equals(1.0) || diceSimilarity.equals(1.0) || jaccardSimilarity.equals(1.0) || actionSize == 0){ +// writer.write(currentLine); +// writer.write("\n"); +// } +// } +// writer.close(); +// } catch (FileNotFoundException e) { +// log.error("File not found"); +// e.printStackTrace(); +// } catch (IOException e) { +// log.error("Error initializing stream"); +// e.printStackTrace(); +// +// } +// } private static void coreLoop(File mes, String outputPath,String inputPath) { try { @@ -391,7 +462,7 @@ public class MultiThreadTreeLoader { } static final JedisPoolConfig poolConfig = buildPoolConfig(); - static JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",6379,20000000); + private static JedisPoolConfig buildPoolConfig() { final JedisPoolConfig poolConfig = new JedisPoolConfig();