diff --git a/pom.xml b/pom.xml index 5c9389c..1378d5b 100644 --- a/pom.xml +++ b/pom.xml @@ -108,7 +108,7 @@ - edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader + edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/AkkaTreeLoader.java new file mode 100644 index 0000000..bdbcef0 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/AkkaTreeLoader.java @@ -0,0 +1,651 @@ +package edu.lu.uni.serval.FixPatternParser.violations; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import akka.actor.Props; +import com.github.gumtreediff.actions.ActionGenerator; +import com.github.gumtreediff.actions.model.*; +import com.github.gumtreediff.matchers.Matcher; +import com.github.gumtreediff.matchers.Matchers; +import com.github.gumtreediff.tree.ITree; +import com.github.gumtreediff.tree.Tree; +import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap; +import edu.lu.uni.serval.MultipleThreadsParser.*; +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import edu.lu.uni.serval.utils.FileHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.*; + +import java.io.*; +import java.nio.file.Files; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.Executors; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Created by anilkoyuncu on 19/03/2018. + */ +public class AkkaTreeLoader { + + private static class StreamGobbler implements Runnable { + private InputStream inputStream; + private Consumer consumer; + + public StreamGobbler(InputStream inputStream, Consumer consumer) { + this.inputStream = inputStream; + this.consumer = consumer; + } + + @Override + public void run() { + new BufferedReader(new InputStreamReader(inputStream)).lines() + .forEach(consumer); + } + } + + private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class); + + + public static void loadRedis(String cmd,String serverWait){ + Process process; + + try { +// String comd = String.format(cmd, f.getAbsoluteFile()); + process = Runtime.getRuntime() + + .exec(cmd); + + + StreamGobbler streamGobbler = + new StreamGobbler(process.getInputStream(), System.out::println); + Executors.newSingleThreadExecutor().submit(streamGobbler); +// int exitCode = process.waitFor(); +// assert exitCode == 0; + Thread.sleep(Integer.valueOf(serverWait)); + + } catch (IOException e) { + e.printStackTrace(); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + log.info("Load done"); + } + + public static void main(String[] args) { + + + String inputPath; + String outputPath; + String port; + String portInner; + String pairsCSVPath; + String importScript; + String pairsCompletedPath; + String serverWait; + String option; + String dbDir; + String chunkName; + String numOfWorkers; + if (args.length > 0) { + inputPath = args[0]; + port = args[1]; + portInner = args[2]; + serverWait = args[3]; + option = args[4]; + chunkName = args[5]; + dbDir = args[6]; + numOfWorkers = args[7]; +// pairsCSVPath = args[3]; +// importScript = args[4]; +// pairsCompletedPath = args[3]; + } else { + inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; + outputPath = "/Users/anilkoyuncu/bugStudy/dataset/"; + port = "6379"; + portInner = "6380"; + serverWait = "10000"; + option = "COMP"; + pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test"; + importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh"; + pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed"; + chunkName ="chunk5.rdb"; + dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; + numOfWorkers = "16"; + } + + if (option.equals("CALC")) { + calculatePairs(inputPath, port); + log.info("Calculate pairs done"); + }else { + comparePairs(port, inputPath, portInner, serverWait,chunkName,dbDir,numOfWorkers); + } + + + } + + + public static void comparePairs(String port,String inputPath, String innerPort,String serverWait,String chunkName, String dbDir,String numOfWorkers){ +// String cmd; +// cmd = "bash " + importScript +" %s"; + + + + List dir; + List path; + + String orgDbname; + + + File files = new File(dbDir); + File[] listFiles = files.listFiles(); + + + Stream stream = Arrays.stream(listFiles); + List folders = stream + .filter(x -> x.getName().equals(chunkName)) + .collect(Collectors.toList()); + for (File folder : folders) { + + + + + String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; + cmd = String.format(cmd, dbDir,folder.getName(),Integer.valueOf(innerPort)); + loadRedis(cmd,serverWait); + + + JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000); + ScanResult scan; + try (Jedis inner = pool.getResource()) { + while (inner.ping()== "PONG"){ + log.info("wait"); + } + + ScanParams sc = new ScanParams(); + sc.count(150000000); + sc.match("pair_*"); + + scan = inner.scan("0", sc); + int size = scan.getResult().size(); + log.info("Scanning " + String.valueOf(size)); + } + List result = scan.getResult(); + + + ActorSystem system = null; + ActorRef parsingActor = null; + final WorkMessage msg = new WorkMessage(0, result,innerPort,inputPath,dbDir,serverWait); + try { + log.info("Akka begins..."); + system = ActorSystem.create("Tree-System"); + parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "tree-actor"); + parsingActor.tell(msg, ActorRef.noSender()); + } catch (Exception e) { + system.shutdown(); + e.printStackTrace(); + } +// greeter.tell(); + +// .parallelStream() +// .forEach(m -> coreCompare(m, inputPath, innerPort)); + + +// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; +// stopServer = String.format(stopServer,Integer.valueOf(innerPort)); +// loadRedis(stopServer,serverWait); + + } + + + + + } + + private static void coreCompare(String name , String inputPath, String innerPort) { + JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000); + Map resultMap; + try (Jedis jedis = pool.getResource()) { + resultMap = jedis.hgetAll(name); + } + String[] split = name.split("_"); + + + String i = split[1]; + String j = split[2]; + String firstValue = resultMap.get("0"); + String secondValue = resultMap.get("1"); + + String[] firstValueSplit = firstValue.split("GumTreeOutput2"); + String[] secondValueSplit = secondValue.split("GumTreeOutput2"); + + if (firstValueSplit.length == 1) { + firstValue = inputPath + firstValueSplit[0]; + } else { + firstValue = inputPath + firstValueSplit[1]; + } + + if (secondValueSplit.length == 1) { + secondValue = inputPath + secondValueSplit[0]; + } else { + secondValue = inputPath + secondValueSplit[1]; + } + + try { + ITree oldTree = getSimpliedTree(firstValue); + + ITree newTree = getSimpliedTree(secondValue); + + Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); + m.match(); + + + ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); + ag.generate(); + List actions = ag.getActions(); + + double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree); + String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1); + double diceSimilarity1 = m.diceSimilarity(oldTree, newTree); + String diceSimilarity = String.format("%1.2f", diceSimilarity1); + double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree); + String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); + + String editDistance = String.valueOf(actions.size()); + + String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; + + + if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) + || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { + String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); + + try (Jedis jedis = pool.getResource()) { + jedis.select(1); + jedis.set(matchKey, result); + } + } + + + try (Jedis jedis = pool.getResource()) { + jedis.del("pair_"+ (String.valueOf(i)) + "_" + String.valueOf(j)); + } + + + + + }catch (Exception e){ + log.error(e.toString() + " {}",(name)); + + + } + + + + + + } + + + + public static void calculatePairs(String inputPath,String port) { + File folder = new File(inputPath); + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List pjs = stream + .filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); + + List fileToCompare = new ArrayList<>(); + for (File pj : pjs) { + File[] files = pj.listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + return name.startsWith("ActionSetDumps"); + } + }); + Collections.addAll(fileToCompare, files[0].listFiles()); + + } + System.out.println("a"); +// compareAll(fileToCompare); + readMessageFiles(fileToCompare,port); + } + + public static void processMessages(String inputPath, String outputPath) { + File folder = new File(outputPath + "pairs_splitted/"); + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List pjs = stream + .filter(x -> !x.getName().startsWith(".")) + .collect(Collectors.toList()); + FileHelper.createDirectory(outputPath + "comparison_splitted/"); + pjs.parallelStream() + .forEach(m -> coreLoop(m, outputPath,inputPath)); + } + + + public static ITree getSimpliedTree(String fn) { + HierarchicalActionSet actionSet = null; + try { + FileInputStream fi = new FileInputStream(new File(fn)); + ObjectInputStream oi = new ObjectInputStream(fi); + actionSet = (HierarchicalActionSet) oi.readObject(); + oi.close(); + fi.close(); + + + } catch (FileNotFoundException e) { + log.error("File not found"); + e.printStackTrace(); + } catch (IOException e) { + log.error("Error initializing stream"); + e.printStackTrace(); + } catch (ClassNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + ITree parent = null; + ITree children =null; + ITree tree = getASTTree(actionSet, parent, children); + tree.setParent(null); + + return tree; + + } + + public static List getKeysByValue(Map map, E value) { + return map.entrySet() + .stream() + .filter(entry -> Objects.equals(entry.getValue(), value)) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } + + public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children){ + + int newType = 0; + + String astNodeType = actionSet.getAstNodeType(); + List keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType); + + if(keysByValue.size() != 1){ + log.error("Birden cok astnodemapmapping"); + } + newType = keysByValue.get(0); + if(actionSet.getParent() == null){ + //root + + parent = new Tree(newType,""); + }else{ + children = new Tree(newType,""); + parent.addChild(children); + } + List subActions = actionSet.getSubActions(); + if (subActions.size() != 0){ + for (HierarchicalActionSet subAction : subActions) { + + if(actionSet.getParent() == null){ + children = parent; + } + getASTTree(subAction,children,null); + + } + + + } + return parent; + } + + public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children){ + + int newType = 0; + + Action action = actionSet.getAction(); + if (action instanceof Update){ + newType = 101; + }else if(action instanceof Insert){ + newType =100; + }else if(action instanceof Move){ + newType = 102; + }else if(action instanceof Delete){ + newType=103; + }else{ + new Exception("unknow action"); + } + if(actionSet.getParent() == null){ + //root + + parent = new Tree(newType,""); + }else{ + children = new Tree(newType,""); + parent.addChild(children); + } + List subActions = actionSet.getSubActions(); + if (subActions.size() != 0){ + for (HierarchicalActionSet subAction : subActions) { + + if(actionSet.getParent() == null){ + children = parent; + } + getActionTree(subAction,children,null); + + } + + + } + return parent; + } + + + + private static void coreLoop(File mes, String outputPath,String inputPath) { + try { + + log.info("Starting in coreLoop"); + + BufferedReader br = null; + String sCurrentLine = null; + BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "comparison_splitted/" + "output_" + mes.getName())); + + br = new BufferedReader( + new FileReader(mes)); + while ((sCurrentLine = br.readLine()) != null) { + String currentLine = sCurrentLine; + String[] split = currentLine.split("\t"); + String i = split[0]; + String j = split[1]; + String firstValue = split[2]; + String secondValue = split[3]; + + firstValue = inputPath + firstValue.split("GumTreeOutput2")[1]; + secondValue = inputPath + secondValue.split("GumTreeOutput2")[1]; + + ITree oldTree = getSimpliedTree(firstValue); + + ITree newTree = getSimpliedTree(secondValue); + + Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); + m.match(); + + ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); + ag.generate(); + List actions = ag.getActions(); + writer.write(String.valueOf(i)); + writer.write("\t"); + writer.write(String.valueOf(j)); + writer.write("\t"); + + writer.write(String.format("%1.2f", m.chawatheSimilarity(oldTree, newTree))); + writer.write("\t"); + writer.write(String.format("%1.2f", m.diceSimilarity(oldTree, newTree))); + writer.write("\t"); + writer.write(String.format("%1.2f", m.jaccardSimilarity(oldTree, newTree))); + writer.write("\t"); + writer.write(String.valueOf(actions.size())); + writer.write("\t"); + writer.write(firstValue); + writer.write("\t"); + writer.write(secondValue); + writer.write("\n"); + + + } + writer.close(); + } catch (FileNotFoundException e) { + log.error("File not found"); + e.printStackTrace(); + } catch (IOException e) { + log.error("Error initializing stream"); + e.printStackTrace(); + + } + log.info("Completed output_" + mes.getName()); + } + + private static void readMessageFiles(List folders,String port) { + + List treesFileNames = new ArrayList<>(); + + + for (File target : folders) { + + treesFileNames.add(target.toString()); + } +// FileHelper.createDirectory(outputPath + "pairs/"); + log.info("Calculating pairs"); +// treesFileNames = treesFileNames.subList(0,100); + byte [] buf = new byte[0]; + String line = null; + + +// FileChannel rwChannel = new RandomAccessFile(outputPath + "pairs/" +"textfile.txt", "rw").getChannel(); +// ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE); + int fileCounter = 0; + + JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); + try (Jedis jedis = jedisPool.getResource()) { + List dir = null; + List path = null; + for (int i = 0; i < treesFileNames.size(); i++) { + for (int j = i + 1; j < treesFileNames.size(); j++) { + + + // do operations with jedis resource + + String key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j); +// String value = treesFileNames.get(i).split("GumTreeOutput2")[1] +","+treesFileNames.get(j).split("GumTreeOutput2")[1]; +// jedis.set(key,value); + + jedis.hset(key,"0",treesFileNames.get(i).split("GumTreeOutput2")[1]); + jedis.hset(key,"1",treesFileNames.get(j).split("GumTreeOutput2")[1]); + //10000000 + if(Integer.compare(jedis.dbSize().intValue(),10000000) == 0){ + dir = jedis.configGet("dir"); + path = jedis.configGet("dbfilename"); + File dbPath = new File(dir.get(1)+"/"+path.get(1)); + File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb"); + try { + jedis.save(); + while (jedis.ping()== "PONG"){ + log.info("wait"); + } + + + + Files.copy(dbPath.toPath(),savePath.toPath()); + } catch (IOException e) { + + e.printStackTrace(); + } + fileCounter++; + jedis.flushDB(); + + } + + + } + } + jedis.save(); + fileCounter++; + File dbPath = new File(dir.get(1)+"/"+path.get(1)); + File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb"); + try { + + while (jedis.ping()== "PONG"){ + log.info("wait"); + } + + + + Files.copy(dbPath.toPath(),savePath.toPath()); + } catch (IOException e) { + + e.printStackTrace(); + } + jedis.flushDB(); + } + + + + + log.info("Done pairs"); + } + + static final JedisPoolConfig poolConfig = buildPoolConfig(); + + + private static JedisPoolConfig buildPoolConfig() { + final JedisPoolConfig poolConfig = new JedisPoolConfig(); + poolConfig.setMaxTotal(128); + poolConfig.setMaxIdle(128); + poolConfig.setMinIdle(16); + poolConfig.setTestOnBorrow(true); + poolConfig.setTestOnReturn(true); + poolConfig.setTestWhileIdle(true); + poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); + poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); + poolConfig.setNumTestsPerEvictionRun(3); + poolConfig.setBlockWhenExhausted(true); + + return poolConfig; + } + + private static List getMessageFiles(String gumTreeInput) { + String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile + File revFilesPath = new File(inputPath + "revFiles/"); + File[] revFiles = revFilesPath.listFiles(); // project folders + List msgFiles = new ArrayList<>(); + if (revFiles.length >= 0) { + for (File revFile : revFiles) { +// if (revFile.getName().endsWith(".java")) { + String fileName = revFile.getName(); + File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file + fileName = fileName.replace(".java", ".txt"); + File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file + File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file + edu.lu.uni.serval.MultipleThreadsParser.MessageFile msgFile = new edu.lu.uni.serval.MultipleThreadsParser.MessageFile(revFile, prevFile, diffentryFile); + msgFile.setPositionFile(positionFile); + msgFiles.add(msgFile); +// } + } + + return msgFiles; + } + else{ + return null; + } + } + + + +// return msgFiles; +} + + + + diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/Compare.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/Compare.java new file mode 100644 index 0000000..6c4d11b --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/Compare.java @@ -0,0 +1,102 @@ +package edu.lu.uni.serval.FixPatternParser.violations; + +import com.github.gumtreediff.actions.ActionGenerator; +import com.github.gumtreediff.actions.model.Action; +import com.github.gumtreediff.matchers.Matcher; +import com.github.gumtreediff.matchers.Matchers; +import com.github.gumtreediff.tree.ITree; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; + +import java.util.List; +import java.util.Map; + +import static edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader.getSimpliedTree; + +/** + * Created by anilkoyuncu on 03/04/2018. + */ +public class Compare { + + private Logger log = LoggerFactory.getLogger(Compare.class); + + public void coreCompare(String name , String inputPath, String innerPort) { + JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000); + Map resultMap; + try (Jedis jedis = pool.getResource()) { + resultMap = jedis.hgetAll(name); + } + String[] split = name.split("_"); + + + String i = split[1]; + String j = split[2]; + String firstValue = resultMap.get("0"); + String secondValue = resultMap.get("1"); + + String[] firstValueSplit = firstValue.split("GumTreeOutput2"); + String[] secondValueSplit = secondValue.split("GumTreeOutput2"); + + if (firstValueSplit.length == 1) { + firstValue = inputPath + firstValueSplit[0]; + } else { + firstValue = inputPath + firstValueSplit[1]; + } + + if (secondValueSplit.length == 1) { + secondValue = inputPath + secondValueSplit[0]; + } else { + secondValue = inputPath + secondValueSplit[1]; + } + + try { + ITree oldTree = getSimpliedTree(firstValue); + + ITree newTree = getSimpliedTree(secondValue); + + Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); + m.match(); + + + ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); + ag.generate(); + List actions = ag.getActions(); + + double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree); + String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1); + double diceSimilarity1 = m.diceSimilarity(oldTree, newTree); + String diceSimilarity = String.format("%1.2f", diceSimilarity1); + double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree); + String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); + + String editDistance = String.valueOf(actions.size()); + + String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; + + + if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) + || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { + String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); + log.info(matchKey); + try (Jedis jedis = pool.getResource()) { + jedis.select(1); + jedis.set(matchKey, result); + } + } + + + try (Jedis jedis = pool.getResource()) { + jedis.del("pair_" + (String.valueOf(i)) + "_" + String.valueOf(j)); + } + + + } catch (Exception e) { + log.error(e.toString() + " {}", (name)); + + + } + } + } diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/Message.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/Message.java new file mode 100644 index 0000000..969a4c2 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/Message.java @@ -0,0 +1,20 @@ +package edu.lu.uni.serval.FixPatternParser.violations; + +import java.io.File; + +public class Message { + + private String name; + private String inputPath; + private String innerPort; + + public Message(String name , String inputPath, String innerPort) { + super(); + this.name = name; + this.inputPath = inputPath; + this.innerPort = innerPort; + } + + + +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java index ff550f9..a30d3fc 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoader.java @@ -111,7 +111,7 @@ public class MultiThreadTreeLoader { pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test"; importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh"; pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed"; - chunkName ="chunk0.rdb"; + chunkName ="chunk5.rdb"; dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; } diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/RunnableCompare.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/RunnableCompare.java new file mode 100644 index 0000000..0ff319f --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/RunnableCompare.java @@ -0,0 +1,25 @@ +package edu.lu.uni.serval.FixPatternParser.violations; + +import edu.lu.uni.serval.FixPatternParser.Parser; + +import java.io.File; + +public class RunnableCompare implements Runnable { + + private String name; + private String inputPath; + private String innerPort; + private Compare comparer; + + public RunnableCompare(String name , String inputPath, String innerPort, Compare comp) { + this.name = name; + this.inputPath = inputPath; + this.innerPort = innerPort; + this.comparer = comp; + } + + @Override + public void run() { + comparer.coreCompare(name, inputPath, innerPort); + } +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeActor.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeActor.java new file mode 100644 index 0000000..a72c1f0 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeActor.java @@ -0,0 +1,82 @@ +package edu.lu.uni.serval.FixPatternParser.violations; + +import akka.actor.ActorRef; +import akka.actor.Props; +import akka.actor.UntypedActor; +import akka.japi.Creator; +import akka.routing.RoundRobinPool; + +import edu.lu.uni.serval.FixPatternParser.violations.WorkMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class TreeActor extends UntypedActor { + + private static Logger logger = LoggerFactory.getLogger(TreeActor.class); + + private ActorRef mineRouter; + private final int numberOfWorkers; + private int counter = 0; + + public TreeActor(int numberOfWorkers) { + mineRouter = this.getContext().actorOf(new RoundRobinPool(numberOfWorkers) + .props(TreeWorker.props()), "tree-router"); + this.numberOfWorkers = numberOfWorkers; + } + + public static Props props(final int numberOfWorkers) { + + return Props.create(new Creator() { + + private static final long serialVersionUID = 9207427376110704705L; + + @Override + public TreeActor create() throws Exception { + return new TreeActor(numberOfWorkers); + } + + }); + } + + @SuppressWarnings("deprecation") + @Override + public void onReceive(Object message) throws Exception { + if (message instanceof WorkMessage) { + List files = ((WorkMessage) message).getMsgFiles(); + String innerPort = ((WorkMessage) message).getInnerPort(); + String inputPath = ((WorkMessage) message).getInputPath(); + String dbDir = ((WorkMessage) message).getDbDir(); + String serverWait = ((WorkMessage) message).getServerWait(); + + int size = files.size(); + int average = size / numberOfWorkers; + int reminder = size % numberOfWorkers; + int counter = 0; + + for (int i = 0; i < numberOfWorkers; i ++) { + int fromIndex = i * average + counter; + if (counter < reminder) counter ++; + int toIndex = (i + 1) * average + counter; + + List filesOfWorkers = files.subList(fromIndex, toIndex); + final WorkMessage workMsg = new WorkMessage(i + 1, filesOfWorkers,innerPort,inputPath,dbDir,serverWait); + mineRouter.tell(workMsg, getSelf()); + logger.info("Assign a task to worker #" + (i + 1) + "..."); + } + } else if ("STOP".equals(message.toString())) { + counter ++; + logger.info(counter + " workers finailized their work..."); + if (counter >= numberOfWorkers) { + logger.info("All workers finailized their work..."); + this.getContext().stop(mineRouter); + this.getContext().stop(getSelf()); + this.getContext().system().shutdown(); + } + } else { + unhandled(message); + } + } + +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeWorker.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeWorker.java new file mode 100644 index 0000000..f9ae8a4 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TreeWorker.java @@ -0,0 +1,206 @@ +package edu.lu.uni.serval.FixPatternParser.violations; + +import akka.actor.Props; +import akka.actor.UntypedActor; +import akka.japi.Creator; +import com.github.gumtreediff.actions.ActionGenerator; +import com.github.gumtreediff.actions.model.Action; +import com.github.gumtreediff.matchers.Matcher; +import com.github.gumtreediff.matchers.Matchers; +import com.github.gumtreediff.tree.ITree; +import edu.lu.uni.serval.FixPatternParser.RunnableParser; +import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; + +import edu.lu.uni.serval.FixPatternParser.violations.WorkMessage; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.utils.FileHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; + +import static edu.lu.uni.serval.FixPatternParser.violations.AkkaTreeLoader.loadRedis; +import static edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoader.getSimpliedTree; + +public class TreeWorker extends UntypedActor { + private static Logger log = LoggerFactory.getLogger(TreeWorker.class); + + + + public TreeWorker() { + + + } + + public static Props props() { + return Props.create(new Creator() { + + private static final long serialVersionUID = -7615153844097275009L; + + @Override + public TreeWorker create() throws Exception { + return new TreeWorker(); + } + + }); + } + + @Override + public void onReceive(Object message) throws Exception { + if(message instanceof edu.lu.uni.serval.FixPatternParser.violations.WorkMessage) { + + log.info("do sometihng"); +// if (message instanceof edu.lu.uni.serval.MultipleThreadsParser.WorkMessage) { + edu.lu.uni.serval.FixPatternParser.violations.WorkMessage msg = (WorkMessage) message; + List files = msg.getMsgFiles(); + String innerPort = msg.getInnerPort(); + String inputPath = msg.getInputPath(); + String dbDir = msg.getDbDir(); + String serverWait = msg.getServerWait(); +// StringBuilder editScripts = new StringBuilder(); +// StringBuilder patchesSourceCode = new StringBuilder(); +// StringBuilder sizes = new StringBuilder(); +// StringBuilder tokens = new StringBuilder(); +// StringBuilder testingInfo = new StringBuilder(); +// +// int id = msg.getId(); +// int counter = 0; +// +// int nullGumTreeResults = 0; +// int noSourceCodeChanges = 0; +// int noStatementChanges = 0; +// int nullDiffEntry = 0; +// int nullMappingGumTreeResults = 0; +// int pureDeletion = 0; +// int largeHunk = 0; +// int nullSourceCode = 0; +// int testInfos = 0; +// int timeouts = 0; +// StringBuilder builder = new StringBuilder(); +// + for (String name : files) { + +// log.info(name); +// File revFile = msgFile.getRevFile(); +// File prevFile = msgFile.getPrevFile(); +// File diffentryFile = msgFile.getDiffEntryFile(); +//// File positionFile = msgFile.getPositionFile(); +// /*if (revFile.getName().toLowerCase().contains("test")) { +// continue; +// }*/ +// FixedViolationHunkParser parser = new FixedViolationHunkParser(); +// + final ExecutorService executor = Executors.newSingleThreadExecutor(); +// // schedule the work + final Future future = executor.submit(new RunnableCompare(name, inputPath, innerPort, new Compare())); + try { + // wait for task to complete + future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); +// +// nullDiffEntry += parser.nullMatchedDiffEntry; +// nullMappingGumTreeResults += parser.nullMappingGumTreeResult; +// pureDeletion += parser.pureDeletions; +// largeHunk += parser.largeHunk; +// nullSourceCode += parser.nullSourceCode; +// testInfos += parser.testInfos; +// testingInfo.append(parser.testingInfo); +// builder.append(parser.unfixedViolations); +// +// String editScript = parser.getAstEditScripts(); +// if ("".equals(editScript)) { +//// if (parser.resultType == 1) { +//// nullGumTreeResults += countAlarms(positionFile, ""); +//// } else if (parser.resultType == 2) { +//// noSourceCodeChanges += countAlarms(positionFile, ""); +//// } else if (parser.resultType == 3) { +//// noStatementChanges += countAlarms(positionFile, ""); +//// } +// } else { +// editScripts.append(editScript); +// patchesSourceCode.append(parser.getPatchesSourceCode()); +// sizes.append(parser.getSizes()); +// tokens.append(parser.getTokensOfSourceCode()); +// +// counter ++; +// if (counter % 100 == 0) { +// FileHelper.outputToFile(editScriptsFilePath + "edistScripts_" + id + ".list", editScripts, true); +// FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true); +// FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true); +// FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true); +// editScripts.setLength(0); +// patchesSourceCode.setLength(0); +// sizes.setLength(0); +// tokens.setLength(0); +// log.info("Worker #" + id +" finialized parsing " + counter + " files..."); +// FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); +// testingInfo.setLength(0); +// } +// } + } catch (TimeoutException e) { + future.cancel(true); +//// timeouts += countAlarms(positionFile, "#Timeout:"); +// System.err.println("#Timeout: " + revFile.getName()); + } catch (InterruptedException e) { +//// timeouts += countAlarms(positionFile, "#TimeInterrupted:"); +// System.err.println("#TimeInterrupted: " + revFile.getName()); + e.printStackTrace(); + } catch (ExecutionException e) { +//// timeouts += countAlarms(positionFile, "#TimeAborted:"); +// System.err.println("#TimeAborted: " + revFile.getName()); + e.printStackTrace(); + } finally { + executor.shutdownNow(); + } + } + String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; + stopServer = String.format(stopServer,Integer.valueOf(innerPort)); + loadRedis(stopServer,serverWait); +// +// if (sizes.length() > 0) { +// FileHelper.outputToFile(editScriptsFilePath + "editScripts_" + id + ".list", editScripts, true); +// FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true); +// FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true); +// FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true); +// editScripts.setLength(0); +// patchesSourceCode.setLength(0); +// sizes.setLength(0); +// tokens.setLength(0); +// +// FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); +// testingInfo.setLength(0); +// } +// String statistic = "\nNullGumTreeResults: " + nullGumTreeResults + "\nNoSourceCodeChanges: " + noSourceCodeChanges + +// "\nNoStatementChanges: " + noStatementChanges + "\nNullDiffEntry: " + nullDiffEntry + "\nNullMatchedGumTreeResults: " + nullMappingGumTreeResults + +// "\nPureDeletion: " + pureDeletion + "\nLargeHunk: " + largeHunk + "\nNullSourceCode: " + nullSourceCode + +// "\nTestingInfo: " + testInfos + "\nTimeout: " + timeouts; +// FileHelper.outputToFile("OUTPUT/statistic_" + id + ".list", statistic, false); +// FileHelper.outputToFile("OUTPUT/UnfixedV_" + id + ".list", builder, false); +// +// log.info("Worker #" + id +"finialized parsing " + counter + " files..."); +// log.info("Worker #" + id + " finialized the work..."); +// this.getSender().tell("STOP", getSelf()); +// } else { + }else{ + unhandled(message); + } + } + + + + + + + + + + +} diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/WorkMessage.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/WorkMessage.java new file mode 100644 index 0000000..6343c51 --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/WorkMessage.java @@ -0,0 +1,53 @@ +package edu.lu.uni.serval.FixPatternParser.violations; + +import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; + +import java.util.List; + +public class WorkMessage { + + private int id; + private List msgFiles; + + public String getDbDir() { + return dbDir; + } + + public String getServerWait() { + return serverWait; + } + + private String dbDir; + private String serverWait; + + public String getInputPath() { + return inputPath; + } + + private String inputPath; + + public String getInnerPort() { + return innerPort; + } + + private String innerPort; + + public WorkMessage(int id, List msgFiles,String innerPort,String inputPath,String dbDir,String serverWait) { + super(); + this.id = id; + this.msgFiles = msgFiles; + this.innerPort = innerPort; + this.inputPath = inputPath; + this.dbDir = dbDir; + this.serverWait = serverWait; + } + + public int getId() { + return id; + } + + public List getMsgFiles() { + return msgFiles; + } + +} \ No newline at end of file