diff --git a/src/main/java/edu/lu/uni/serval/config/Configuration.java b/src/main/java/edu/lu/uni/serval/config/Configuration.java deleted file mode 100644 index 4421c72..0000000 --- a/src/main/java/edu/lu/uni/serval/config/Configuration.java +++ /dev/null @@ -1,7 +0,0 @@ -package edu.lu.uni.serval.config; - -public class Configuration { - - public static final long SECONDS_TO_WAIT = 900L; - -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java index 2dec952..a4cbbd0 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java @@ -39,23 +39,24 @@ public class Launcher { String chunk = appProps.getProperty("chunk","1.txt"); String eDiffTimeout = appProps.getProperty("eDiffTimeout","900"); String isBig = appProps.getProperty("isBigPair","true"); + String parallelism = appProps.getProperty("parallelism","FORKJOIN"); boolean isBigPair = Boolean.parseBoolean(isBig); String parameters = String.format("\nportInner %s " + "\nnumOfWorkers %s " + "\njobType %s \nport %s " + "\npythonPath %s \ndatasetPath %s" + - "\npjName %s \nactionType %s \nthreshold %s \ncursor %s \neDiffTimeout %s \nisBigPair %s" - , portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,eDiffTimeout,isBig); + "\npjName %s \nactionType %s \nthreshold %s \ncursor %s \neDiffTimeout %s \nisBigPair %s \nparallelism %s" + , portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,eDiffTimeout,isBig,parallelism); log.info(parameters); - mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk,eDiffTimeout,isBigPair); + mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk,eDiffTimeout,isBigPair,parallelism); } - public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk, String eDiffTimeout, boolean isBigPair){ + public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk, String eDiffTimeout, boolean isBigPair, String parallelism){ String dbDir; @@ -74,17 +75,17 @@ public class Launcher { try { switch (jobType) { case "ENHANCEDASTDIFF": - EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName, eDiffTimeout,actionType); + EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName, eDiffTimeout,actionType,parallelism); break; case "CACHE": StoreEDiffInCache.main(gumOutput, portDumps, dbDir, actionType+dumpsName,actionType); break; case "SI": CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType,isBigPair,iCursor); - ImportPairs2DB.main(pairsPath+actionType, portInner, dbDir,datasetPath); + ImportPairs2DB.main(pairsPath+actionType, portInner, dbDir,datasetPath,chunk); break; case "SIMI": - AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,iCursor,chunk,eDiffTimeout); + AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,iCursor,chunk,eDiffTimeout,parallelism); break; case "LEVEL1": diff --git a/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java deleted file mode 100644 index 8734393..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java +++ /dev/null @@ -1,600 +0,0 @@ -package edu.lu.uni.serval.fixminer; - -import com.github.gumtreediff.actions.ActionGenerator; -import com.github.gumtreediff.actions.model.Action; -import com.github.gumtreediff.matchers.Matcher; -import com.github.gumtreediff.matchers.Matchers; -import com.github.gumtreediff.tree.ITree; -import com.github.gumtreediff.tree.Tree; -import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; -import edu.lu.uni.serval.utils.ASTNodeMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.*; - -import java.io.*; -import java.nio.file.Files; -import java.time.Duration; -import java.util.*; -import java.util.concurrent.Executors; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Created by anilkoyuncu on 19/03/2018. - */ -public class MultiThreadTreeLoader { - - private static class StreamGobbler implements Runnable { - private InputStream inputStream; - private Consumer consumer; - - public StreamGobbler(InputStream inputStream, Consumer consumer) { - this.inputStream = inputStream; - this.consumer = consumer; - } - - @Override - public void run() { - new BufferedReader(new InputStreamReader(inputStream)).lines() - .forEach(consumer); - } - } - - private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoader.class); - - - public static void loadRedis(String cmd,String serverWait){ - Process process; - - try { -// String comd = String.format(cmd, f.getAbsoluteFile()); - process = Runtime.getRuntime() - - .exec(cmd); - - - StreamGobbler streamGobbler = - new StreamGobbler(process.getInputStream(), System.out::println); - Executors.newSingleThreadExecutor().submit(streamGobbler); -// int exitCode = process.waitFor(); -// assert exitCode == 0; - Thread.sleep(Integer.valueOf(serverWait)); - - } catch (IOException e) { - e.printStackTrace(); - } - catch (InterruptedException e) { - e.printStackTrace(); - } - log.info("Load done"); - } - - public static void main(String[] args) { - - - String inputPath; - String outputPath; - String port; - String portInner; - String pairsCSVPath; - String importScript; - String pairsCompletedPath; - String serverWait; - String option; - String dbDir; - String chunkName; - if (args.length > 0) { - inputPath = args[0]; - port = args[1]; - portInner = args[2]; - serverWait = args[3]; - option = args[4]; - chunkName = args[5]; - dbDir = args[6]; -// pairsCSVPath = args[3]; -// importScript = args[4]; -// pairsCompletedPath = args[3]; - } else { - inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; - outputPath = "/Users/anilkoyuncu/bugStudy/dataset/"; - port = "6379"; - portInner = "6380"; - serverWait = "10000"; - option = "COMP"; - pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test"; - importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh"; - pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed"; - chunkName ="chunk5.rdb"; - dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; - } - - if (option.equals("CALC")) { - calculatePairs(inputPath, port); - log.info("Calculate pairs done"); - }else { - comparePairs(port, inputPath, portInner, serverWait,chunkName,dbDir); - } - - - } - - - public static void comparePairs(String port,String inputPath, String innerPort,String serverWait,String chunkName, String dbDir){ -// String cmd; -// cmd = "bash " + importScript +" %s"; - - - - List dir; - List path; - - String orgDbname; - - - File files = new File(dbDir); - File[] listFiles = files.listFiles(); - - - Stream stream = Arrays.stream(listFiles); - List folders = stream - .filter(x -> x.getName().equals(chunkName)) - .collect(Collectors.toList()); - for (File folder : folders) { - - - - - String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; - cmd = String.format(cmd, dbDir,folder.getName(),Integer.valueOf(innerPort)); - loadRedis(cmd,serverWait); - - - JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000); - ScanResult scan; - try (Jedis inner = pool.getResource()) { - while (inner.ping()== "PONG"){ - log.info("wait"); - } - - ScanParams sc = new ScanParams(); - sc.count(150000000); - sc.match("pair_*"); - - scan = inner.scan("0", sc); - int size = scan.getResult().size(); - log.info("Scanning " + String.valueOf(size)); - } - scan.getResult().parallelStream() - .forEach(m -> coreCompare(m, inputPath, innerPort)); - - - String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; - stopServer = String.format(stopServer,Integer.valueOf(innerPort)); - loadRedis(stopServer,serverWait); - - } - - - - - } - - private static void coreCompare(String name , String inputPath, String innerPort) { - JedisPool pool = new JedisPool(new JedisPoolConfig(), "127.0.0.1", Integer.valueOf(innerPort), 20000000); - Map resultMap; - try (Jedis jedis = pool.getResource()) { - resultMap = jedis.hgetAll(name); - } - String[] split = name.split("_"); - - - String i = split[1]; - String j = split[2]; - String firstValue = resultMap.get("0"); - String secondValue = resultMap.get("1"); - - String[] firstValueSplit = firstValue.split("GumTreeOutput2"); - String[] secondValueSplit = secondValue.split("GumTreeOutput2"); - - if (firstValueSplit.length == 1) { - firstValue = inputPath + firstValueSplit[0]; - } else { - firstValue = inputPath + firstValueSplit[1]; - } - - if (secondValueSplit.length == 1) { - secondValue = inputPath + secondValueSplit[0]; - } else { - secondValue = inputPath + secondValueSplit[1]; - } - - try { - ITree oldTree = getSimpliedTree(firstValue); - - ITree newTree = getSimpliedTree(secondValue); - - Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); - m.match(); - - - ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); - ag.generate(); - List actions = ag.getActions(); - - double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree); - String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1); - double diceSimilarity1 = m.diceSimilarity(oldTree, newTree); - String diceSimilarity = String.format("%1.2f", diceSimilarity1); - double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree); - String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); - - String editDistance = String.valueOf(actions.size()); - - String result = resultMap.get("0") + "," + resultMap.get("1") + "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; - - - if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) - || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { - String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); - - try (Jedis jedis = pool.getResource()) { - jedis.select(1); - jedis.set(matchKey, result); - } - } - - - try (Jedis jedis = pool.getResource()) { - jedis.del("pair_"+ (String.valueOf(i)) + "_" + String.valueOf(j)); - } - - - - - }catch (Exception e){ - log.warn(e.toString() + " {}",(name)); - - - } - - - - - - } - - - - public static void calculatePairs(String inputPath,String port) { - File folder = new File(inputPath); - File[] listOfFiles = folder.listFiles(); - Stream stream = Arrays.stream(listOfFiles); - List pjs = stream - .filter(x -> !x.getName().startsWith(".")) - .collect(Collectors.toList()); - - List fileToCompare = new ArrayList<>(); - for (File pj : pjs) { - File[] files = pj.listFiles(new FilenameFilter() { - public boolean accept(File dir, String name) { - return name.startsWith("ActionSetDumps"); - } - }); - Collections.addAll(fileToCompare, files[0].listFiles()); - - } - System.out.println("a"); -// compareAll(fileToCompare); - readMessageFiles(fileToCompare,port); - } - -// public static void processMessages(String inputPath, String outputPath) { -// File folder = new File(outputPath + "pairs_splitted/"); -// File[] listOfFiles = folder.listFiles(); -// Stream stream = Arrays.stream(listOfFiles); -// List pjs = stream -// .filter(x -> !x.getName().startsWith(".")) -// .collect(Collectors.toList()); -// FileHelper.createDirectory(outputPath + "comparison_splitted/"); -// pjs.parallelStream() -// .forEach(m -> coreLoop(m, outputPath,inputPath)); -// } - - - public static ITree getSimpliedTree(String fn) { - HierarchicalActionSet actionSet = null; - try { - FileInputStream fi = new FileInputStream(new File(fn)); - ObjectInputStream oi = new ObjectInputStream(fi); - actionSet = (HierarchicalActionSet) oi.readObject(); - oi.close(); - fi.close(); - - - } catch (FileNotFoundException e) { - log.error("File not found"); - e.printStackTrace(); - } catch (IOException e) { - log.error("Error initializing stream"); - e.printStackTrace(); - } catch (ClassNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - ITree parent = null; - ITree children =null; - ITree tree = getASTTree(actionSet, parent, children); - tree.setParent(null); - - return tree; - - } - - public static List getKeysByValue(Map map, E value) { - return map.entrySet() - .stream() - .filter(entry -> Objects.equals(entry.getValue(), value)) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - } - - public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children){ - - int newType = 0; - - String astNodeType = actionSet.getAstNodeType(); - List keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType); - - if(keysByValue.size() != 1){ - log.error("Birden cok astnodemapmapping"); - } - newType = keysByValue.get(0); - if(actionSet.getParent() == null){ - //root - - parent = new Tree(newType,""); - }else{ - children = new Tree(newType,""); - parent.addChild(children); - } - List subActions = actionSet.getSubActions(); - if (subActions.size() != 0){ - for (HierarchicalActionSet subAction : subActions) { - - if(actionSet.getParent() == null){ - children = parent; - } - getASTTree(subAction,children,null); - - } - - - } - return parent; - } - -// public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children){ -// -// int newType = 0; -// -// Action action = actionSet.getAction(); -// if (action instanceof Update){ -// newType = 101; -// }else if(action instanceof Insert){ -// newType =100; -// }else if(action instanceof Move){ -// newType = 102; -// }else if(action instanceof Delete){ -// newType=103; -// }else{ -// new Exception("unknow action"); -// } -// if(actionSet.getParent() == null){ -// //root -// -// parent = new Tree(newType,""); -// }else{ -// children = new Tree(newType,""); -// parent.addChild(children); -// } -// List subActions = actionSet.getSubActions(); -// if (subActions.size() != 0){ -// for (HierarchicalActionSet subAction : subActions) { -// -// if(actionSet.getParent() == null){ -// children = parent; -// } -// getActionTree(subAction,children,null); -// -// } -// -// -// } -// return parent; -// } - - - - private static void coreLoop(File mes, String outputPath,String inputPath) { - try { - - log.info("Starting in coreLoop"); - - BufferedReader br = null; - String sCurrentLine = null; - BufferedWriter writer = new BufferedWriter(new FileWriter(outputPath + "comparison_splitted/" + "output_" + mes.getName())); - - br = new BufferedReader( - new FileReader(mes)); - while ((sCurrentLine = br.readLine()) != null) { - String currentLine = sCurrentLine; - String[] split = currentLine.split("\t"); - String i = split[0]; - String j = split[1]; - String firstValue = split[2]; - String secondValue = split[3]; - - firstValue = inputPath + firstValue.split("GumTreeOutput2")[1]; - secondValue = inputPath + secondValue.split("GumTreeOutput2")[1]; - - ITree oldTree = getSimpliedTree(firstValue); - - ITree newTree = getSimpliedTree(secondValue); - - Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); - m.match(); - - ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); - ag.generate(); - List actions = ag.getActions(); - writer.write(String.valueOf(i)); - writer.write("\t"); - writer.write(String.valueOf(j)); - writer.write("\t"); - - writer.write(String.format("%1.2f", m.chawatheSimilarity(oldTree, newTree))); - writer.write("\t"); - writer.write(String.format("%1.2f", m.diceSimilarity(oldTree, newTree))); - writer.write("\t"); - writer.write(String.format("%1.2f", m.jaccardSimilarity(oldTree, newTree))); - writer.write("\t"); - writer.write(String.valueOf(actions.size())); - writer.write("\t"); - writer.write(firstValue); - writer.write("\t"); - writer.write(secondValue); - writer.write("\n"); - - - } - writer.close(); - } catch (FileNotFoundException e) { - log.error("File not found"); - e.printStackTrace(); - } catch (IOException e) { - log.error("Error initializing stream"); - e.printStackTrace(); - - } - log.info("Completed output_" + mes.getName()); - } - - private static void readMessageFiles(List folders,String port) { - - List treesFileNames = new ArrayList<>(); - - - for (File target : folders) { - - treesFileNames.add(target.toString()); - } -// FileHelper.createDirectory(outputPath + "pairs/"); - log.info("Calculating pairs"); -// treesFileNames = treesFileNames.subList(0,100); - byte [] buf = new byte[0]; - String line = null; - - -// FileChannel rwChannel = new RandomAccessFile(outputPath + "pairs/" +"textfile.txt", "rw").getChannel(); -// ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE); - int fileCounter = 0; - - JedisPool jedisPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); - try (Jedis jedis = jedisPool.getResource()) { - List dir = null; - List path = null; - for (int i = 0; i < treesFileNames.size(); i++) { - for (int j = i + 1; j < treesFileNames.size(); j++) { - - - // do operations with jedis resource - - String key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j); -// String value = treesFileNames.get(i).split("GumTreeOutput2")[1] +","+treesFileNames.get(j).split("GumTreeOutput2")[1]; -// jedis.set(key,value); - - jedis.hset(key,"0",treesFileNames.get(i).split("GumTreeOutput2")[1]); - jedis.hset(key,"1",treesFileNames.get(j).split("GumTreeOutput2")[1]); - //10000000 - if(Integer.compare(jedis.dbSize().intValue(),10000000) == 0){ - dir = jedis.configGet("dir"); - path = jedis.configGet("dbfilename"); - File dbPath = new File(dir.get(1)+"/"+path.get(1)); - File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb"); - try { - jedis.save(); - while (jedis.ping()== "PONG"){ - log.info("wait"); - } - - - - Files.copy(dbPath.toPath(),savePath.toPath()); - } catch (IOException e) { - - e.printStackTrace(); - } - fileCounter++; - jedis.flushDB(); - - } - - - } - } - jedis.save(); - fileCounter++; - File dbPath = new File(dir.get(1)+"/"+path.get(1)); - File savePath = new File(dir.get(1)+"/"+"chunk"+String.valueOf(fileCounter)+ ".rdb"); - try { - - while (jedis.ping()== "PONG"){ - log.info("wait"); - } - - - - Files.copy(dbPath.toPath(),savePath.toPath()); - } catch (IOException e) { - - e.printStackTrace(); - } - jedis.flushDB(); - } - - - - - log.info("Done pairs"); - } - - static final JedisPoolConfig poolConfig = buildPoolConfig(); - - - private static JedisPoolConfig buildPoolConfig() { - final JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(128); - poolConfig.setMaxIdle(128); - poolConfig.setMinIdle(16); - poolConfig.setTestOnBorrow(true); - poolConfig.setTestOnReturn(true); - poolConfig.setTestWhileIdle(true); - poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); - poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); - poolConfig.setNumTestsPerEvictionRun(3); - poolConfig.setBlockWhenExhausted(true); - - return poolConfig; - } - - - -// return msgFiles; -} - - - - diff --git a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java deleted file mode 100644 index f116deb..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java +++ /dev/null @@ -1,239 +0,0 @@ -package edu.lu.uni.serval.fixminer; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor; -import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage; -import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile; -import edu.lu.uni.serval.utils.FileHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class TestHunkParserSingleFile { - - private static Logger log = LoggerFactory.getLogger(TestHunkParserSingleFile.class); -// public static void main(String[] args) { -// // input data -// -//// String rootPath = "/Users/anilkoyuncu/bugStudy"; -// String inputPath; -// String outputPath; -// if(args.length > 0){ -// inputPath = args[1]; -// outputPath = args[0]; -// }else{ -//// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeInputBug4"; -// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/allDataset"; -//// outputPath = "/Users/anilkoyuncu/bugStudy/code/python/GumTreeOutput2/"; -// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputSingle"; -// } -// -// //5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.txt_1_CAMEL -// -// File folder = new File(inputPath); -// File[] listOfFiles = folder.listFiles(); -// Stream stream = Arrays.stream(listOfFiles); -// List folders = stream -// .filter(x -> !x.getName().startsWith(".")) -// -// .collect(Collectors.toList()); -// -//// List targetList = new ArrayList(); -//// for (File f:folders){ -//// for(File f1 :f.listFiles()){ -//// if (!f1.getName().startsWith(".")){ -//// targetList.add(f1); -//// } -//// } -//// } -// -//// List pjList = Arrays.asList("DATAJPA","ZXing","PDE","SWS","SWT", "SWF", "COLLECTIONS", "JDT"); -// List files = new ArrayList(); -//// files.add("5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.java"); -// files.add("d6c5e5_9f96d9_hbase-server#src#main#java#org#apache#hadoop#hbase#master#RegionStates.java"); -// for(String f : files){ -// String pjName = "HBASE"; -//// for (File target : folders) { -//// String pjName = target.getName(); -//// if (!pjList.contains(pjName)){ -//// continue; -//// } -// -//// final List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" -// MessageFile messageFile = getMessageFile(inputPath + "/" + pjName +"/", f); -// -// List msgFiles = new ArrayList<>(); -// msgFiles.add(messageFile); -// String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; -// final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list"; -// final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list"; -// final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list"; -// final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv"; -// final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list"; -// -// -// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ActionSetDumps"); -// FileHelper.deleteDirectory(editScriptsFilePath); -// FileHelper.deleteDirectory(patchesSourceCodeFilePath); -// FileHelper.deleteDirectory(buggyTokensFilePath); -// FileHelper.deleteDirectory(editScriptSizesFilePath); -// FileHelper.deleteDirectory(alarmTypesFilePath); -// -// StringBuilder astEditScripts = new StringBuilder(); -// StringBuilder tokens = new StringBuilder(); -// StringBuilder sizes = new StringBuilder(); -// StringBuilder patches = new StringBuilder(); -// StringBuilder alarmTypes = new StringBuilder(); -// -//// int a = 0; -// -// ActorSystem system = null; -// ActorRef parsingActor = null; -// final EDiffMessage msg = new EDiffMessage(0, msgFiles); -// try { -// log.info("Akka begins..."); -// system = ActorSystem.create("Mining-FixPattern-System"); -// parsingActor = system.actorOf(EDiffActor.props(1, "dataset"), "mine-fix-pattern-actor"); -// parsingActor.tell(msg, ActorRef.noSender()); -// } catch (Exception e) { -// system.shutdown(); -// e.printStackTrace(); -// } -// -//// int counter = 0; -//// for (MessageFile msgFile : msgFiles) { -//// EDiffHunkParser parser = new EDiffHunkParser(); -//// -//// final ExecutorService executor = Executors.newSingleThreadExecutor(); -//// // schedule the work -//// final Future future = executor.submit(new RunnableParser(msgFile.getPrevFile(), -//// msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser)); -//// try { -//// // where we wait for task to complete -//// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); -//// String editScripts = parser.getAstEditScripts(); -//// if (!editScripts.equals("")) { -//// astEditScripts.append(editScripts); -//// tokens.append(parser.getTokensOfSourceCode()); -//// sizes.append(parser.getSizes()); -//// patches.append(parser.getPatchesSourceCode()); -//// alarmTypes.append(parser.getAlarmTypes()); -//// -//// a++; -//// if (a % 100 == 0) { -//// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); -//// FileHelper.outputToFile(buggyTokensFilePath, tokens, true); -//// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); -//// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); -//// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); -//// astEditScripts.setLength(0); -//// tokens.setLength(0); -//// sizes.setLength(0); -//// patches.setLength(0); -//// alarmTypes.setLength(0); -//// System.out.println("Finish of parsing " + a + " files......"); -//// } -//// } -//// } catch (TimeoutException e) { -//// err.println("task timed out"); -//// future.cancel(true /* mayInterruptIfRunning */); -//// } catch (InterruptedException e) { -//// err.println("task interrupted"); -//// } catch (ExecutionException e) { -//// err.println("task aborted"); -//// } finally { -//// executor.shutdownNow(); -//// } -//// } -// -// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); -// FileHelper.outputToFile(buggyTokensFilePath, tokens, true); -// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); -// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); -// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); -// astEditScripts.setLength(0); -// tokens.setLength(0); -// sizes.setLength(0); -// patches.setLength(0); -// alarmTypes.setLength(0); -// -// -//// classifyByAlarmTypes(); -// } -// } - - - private static List getMessageFiles(String gumTreeInput) { - String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile - File revFilesPath = new File(inputPath + "revFiles/"); - File[] revFiles = revFilesPath.listFiles(); // project folders - List msgFiles = new ArrayList<>(); - - // gumTreeInput = /Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/AMQP/ - // fileName = 01534a_df5570_spring-rabbit#src#test#java#org#springframework#amqp#rabbit#listener#LocallyTransactedTests.java - if (revFiles.length >= 0) { - for (File revFile : revFiles) { -// if (revFile.getName().endsWith(".java")) { - String fileName = revFile.getName(); - File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file - fileName = fileName.replace(".java", ".txt"); - File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file - File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file - MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); - msgFile.setPositionFile(positionFile); - msgFiles.add(msgFile); -// } - } - - return msgFiles; - } - else{ - return null; - } - } - - private static MessageFile getMessageFile(String gumTreeInput, String fileName) { -// String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile -// File revFilesPath = new File(inputPath + "revFiles/"); -// File[] revFiles = revFilesPath.listFiles(); // project folders -// List msgFiles = new ArrayList<>(); - - // gumTreeInput = /Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/AMQP/ - // fileName = 01534a_df5570_spring-rabbit#src#test#java#org#springframework#amqp#rabbit#listener#LocallyTransactedTests.java -// if (revFiles.length >= 0) { -// for (File revFile : revFiles) { -// if (revFile.getName().endsWith(".java")) { -// String fileName = revFile.getName(); - File revFile = new File(gumTreeInput + "revFiles/"+fileName); - File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file - fileName = fileName.replace(".java", ".txt"); - File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file -// File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file - MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); - return msgFile; -// msgFile.setPositionFile(positionFile); -// msgFiles.add(msgFile); -// } -// } -// -// return msgFiles; -// } -// else{ -// return null; -// } - } - - - - - - - -} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java index aaa6159..15cbc87 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java @@ -19,26 +19,57 @@ public class AkkaTreeParser { private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class); - public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, int cursor, String eDiffTimeout){ + public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, int cursor, String eDiffTimeout, String parallelism){ final List listOfPairs = getMessages(innerPool,cursor); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" + switch (parallelism){ + case "AKKA": + ActorSystem system = null; + ActorRef parsingActor = null; + final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,eDiffTimeout); + try { + log.info("Akka begins..."); + system = ActorSystem.create("Compare-EnhancedDiff-System"); + parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor"); + parsingActor.tell(msg, ActorRef.noSender()); + } catch (Exception e) { + system.shutdown(); + e.printStackTrace(); + } + break; + case "FORKJOIN": + int counter = new Object() { + int counter = 0; - ActorSystem system = null; - ActorRef parsingActor = null; - final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,eDiffTimeout); - try { - log.info("Akka begins..."); - system = ActorSystem.create("Compare-EnhancedDiff-System"); - - parsingActor = system.actorOf(TreeActor.props(Integer.valueOf(numOfWorkers)), "mine-fix-pattern-actor"); - parsingActor.tell(msg, ActorRef.noSender()); - } catch (Exception e) { - system.shutdown(); - e.printStackTrace(); + { + listOfPairs.stream(). + parallel(). + peek(x -> counter++). + forEach(m -> + { + Compare compare = new Compare(); + compare.coreCompare(m, innerPool, outerPool); + if (counter % 10 == 0) { + log.info("Finalized parsing " + counter + " files... remaing " + (listOfPairs.size() - counter)); + } + } + ); + } + }.counter; + log.info("Finished parsing {} files",counter); + break; + default: + log.error("Unknown parallelism {}", parallelism); + break; } + + + + + } public static List getMessages(JedisPool innerPool, int cursor){ diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java index 58337b4..ef93559 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java @@ -5,7 +5,7 @@ import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.matchers.Matcher; import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; -import edu.lu.uni.serval.utils.EDiff; +import edu.lu.uni.serval.utils.EDiffHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -32,7 +32,6 @@ public class Compare { try { jedis = innerPool.getResource(); -// resultMap = jedis.hgetAll(name); String[] split = name.split("_"); @@ -41,12 +40,9 @@ public class Compare { String j = split[2]; -// String firstValue = resultMap.get("0"); -// String secondValue = resultMap.get("1"); + oldTree = EDiffHelper.getSimpliedTree(i,outerPool); - oldTree = EDiff.getSimpliedTree(i,outerPool); - - newTree = EDiff.getSimpliedTree(j,outerPool); + newTree = EDiffHelper.getSimpliedTree(j,outerPool); Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); m.match(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java index 17f660e..24b2045 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java @@ -62,13 +62,13 @@ public class TreeActor extends UntypedActor { List pairsOfWorkers = pairs.subList(fromIndex, toIndex); final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool,((TreeMessage) message).getSECONDS_TO_WAIT()); mineRouter.tell(workMsg, getSelf()); - logger.info("Assign a task to worker #" + (i + 1) + "..."); + logger.info("Assign {} task to worker #" + (i + 1) + "...",pairsOfWorkers.size()); } } else if ("STOP".equals(message.toString())) { counter ++; - logger.info(counter + " workers finailized their work..."); + logger.info(counter + " workers finalized their work..."); if (counter >= numberOfWorkers) { - logger.info("All workers finailized their work..."); + logger.info("All workers finalized their work..."); this.getContext().stop(mineRouter); this.getContext().stop(getSelf()); this.getContext().system().shutdown(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java index 41849f7..ae79022 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java @@ -3,7 +3,6 @@ package edu.lu.uni.serval.fixminer.akka.compare; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; -import edu.lu.uni.serval.config.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.JedisPool; @@ -15,32 +14,7 @@ public class TreeWorker extends UntypedActor { private static Logger log = LoggerFactory.getLogger(TreeWorker.class); -// private JedisPool innerPool; -// private JedisPool outerPool; -// -// public TreeWorker(String innerPort,String outerPort) { -//// this.innerPool = innerPool; -//// this.outerPool = outerPool; -// this.outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(outerPort),20000000); -// this.innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(innerPort),20000000); -// -// -// } -// -// public static Props props(final String innerPort,final String outerPort) { -// return Props.create(new Creator() { -// -// private static final long serialVersionUID = -7615153844097275009L; -// -// @Override -// public TreeWorker create() throws Exception { -// return new TreeWorker(innerPort,outerPort); -// } -// -// }); -// } - - public TreeWorker() { + public TreeWorker() { } @@ -63,160 +37,46 @@ public class TreeWorker extends UntypedActor { public void onReceive(Object message) throws Exception { if(message instanceof TreeMessage) { - -// if (message instanceof edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage) { TreeMessage msg = (TreeMessage) message; List files = msg.getName(); JedisPool innerPool = msg.getInnerPool(); JedisPool outerPool = msg.getOuterPool(); int id = msg.getId(); -// int counter = new Object() { - int counter = 0; + int counter = 0; - // for (String name : files) { -// files.stream(). -// parallel(). -// peek(x -> counter++). -// forEach(m -> -// { -// Compare compare = new Compare(); -// compare.coreCompare(m, innerPool, outerPool); -// } -// ); -// } -// }.counter; -// - final ExecutorService executor = Executors.newWorkStealingPool(); + + Compare compare = new Compare(); + final ExecutorService executor = Executors.newSingleThreadExecutor(); // // schedule the work - final Future future = executor.submit(new RunnableCompare(name, innerPool, outerPool, new Compare())); + final Future future = executor.submit(new RunnableCompare(name, innerPool, outerPool, compare)); try { // wait for task to complete future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS); - Compare compare = new Compare(); - compare.coreCompare(name, innerPool, outerPool); counter++; -// nullDiffEntry += parser.nullMatchedDiffEntry; -// nullMappingGumTreeResults += parser.nullMappingGumTreeResult; -// pureDeletion += parser.pureDeletions; -// largeHunk += parser.largeHunk; -// nullSourceCode += parser.nullSourceCode; -// testInfos += parser.testInfos; -// testingInfo.append(parser.testingInfo); -// builder.append(parser.unfixedViolations); -// -// String editScript = parser.getAstEditScripts(); -// if ("".equals(editScript)) { -//// if (parser.resultType == 1) { -//// nullGumTreeResults += countAlarms(positionFile, ""); -//// } else if (parser.resultType == 2) { -//// noSourceCodeChanges += countAlarms(positionFile, ""); -//// } else if (parser.resultType == 3) { -//// noStatementChanges += countAlarms(positionFile, ""); -//// } -// } else { -// editScripts.append(editScript); -// patchesSourceCode.append(parser.getPatchesSourceCode()); -// sizes.append(parser.getSizes()); -// tokens.append(parser.getTokensOfSourceCode()); -// -// counter ++; -// if (counter % 100 == 0) { -// FileHelper.outputToFile(editScriptsFilePath + "edistScripts_" + id + ".list", editScripts, true); -// FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true); -// FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true); -// FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true); -// editScripts.setLength(0); -// patchesSourceCode.setLength(0); -// sizes.setLength(0); -// tokens.setLength(0); -// log.info("Worker #" + id +" finialized parsing " + counter + " files..."); -// FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); -// testingInfo.setLength(0); -// } -// } + if (counter % 100 == 0) { + log.info("Worker #" + id +" finalized parsing " + counter + " pairs... remaing "+ (files.size() - counter)); + } } catch (TimeoutException e) { future.cancel(true); -////// timeouts += countAlarms(positionFile, "#Timeout:"); -// System.err.println("#Timeout: " + name); } catch (InterruptedException e) { -////// timeouts += countAlarms(positionFile, "#TimeInterrupted:"); -//// System.err.println("#TimeInterrupted: " + revFile.getName()); e.printStackTrace(); } catch (ExecutionException e) { -////// timeouts += countAlarms(positionFile, "#TimeAborted:"); -//// System.err.println("#TimeAborted: " + revFile.getName()); e.printStackTrace(); } finally { executor.shutdownNow(); } } -// log.info("Worker #" + id +"finialized parsing " + counter + " files..."); log.info("Worker #" + id + " finalized the work..."); this.getSender().tell("STOP", getSelf()); -// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; -// stopServer = String.format(stopServer,Integer.valueOf(innerPort)); -// loadRedis(stopServer,serverWait); -// -// if (sizes.length() > 0) { -// FileHelper.outputToFile(editScriptsFilePath + "editScripts_" + id + ".list", editScripts, true); -// FileHelper.outputToFile(patchesSourceCodeFilePath + "patches_" + id + ".list", patchesSourceCode, true); -// FileHelper.outputToFile(editScriptSizesFilePath + "sizes_" + id + ".list", sizes, true); -// FileHelper.outputToFile(buggyTokensFilePath + "tokens_" + id + ".list", tokens, true); -// editScripts.setLength(0); -// patchesSourceCode.setLength(0); -// sizes.setLength(0); -// tokens.setLength(0); -// -// FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); -// testingInfo.setLength(0); -// } -// String statistic = "\nNullGumTreeResults: " + nullGumTreeResults + "\nNoSourceCodeChanges: " + noSourceCodeChanges + -// "\nNoStatementChanges: " + noStatementChanges + "\nNullDiffEntry: " + nullDiffEntry + "\nNullMatchedGumTreeResults: " + nullMappingGumTreeResults + -// "\nPureDeletion: " + pureDeletion + "\nLargeHunk: " + largeHunk + "\nNullSourceCode: " + nullSourceCode + -// "\nTestingInfo: " + testInfos + "\nTimeout: " + timeouts; -// FileHelper.outputToFile("OUTPUT/statistic_" + id + ".list", statistic, false); -// FileHelper.outputToFile("OUTPUT/UnfixedV_" + id + ".list", builder, false); -// -// log.info("Worker #" + id +"finialized parsing " + counter + " files..."); -// log.info("Worker #" + id + " finialized the work..."); -// this.getSender().tell("STOP", getSelf()); -// } else { }else{ unhandled(message); } } -// static final JedisPoolConfig poolConfig = buildPoolConfig(); -// -// -// private static JedisPoolConfig buildPoolConfig() { -// final JedisPoolConfig poolConfig = new JedisPoolConfig(); -// poolConfig.setMaxTotal(128); -// poolConfig.setMaxIdle(128); -// poolConfig.setMinIdle(16); -// poolConfig.setTestOnBorrow(true); -// poolConfig.setTestOnReturn(true); -// poolConfig.setTestWhileIdle(true); -// poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); -// poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); -// poolConfig.setNumTestsPerEvictionRun(3); -// poolConfig.setBlockWhenExhausted(true); -// -// return poolConfig; -// } - - - - - - - - - } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java index 9a354be..5d16315 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java @@ -56,7 +56,7 @@ public class EDiffActor extends UntypedActor { List filesOfWorkers = files.subList(fromIndex, toIndex); final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getActionType()); mineRouter.tell(workMsg, getSelf()); - logger.info("Assign a task to worker #" + (i + 1) + "..."); + logger.info("Assign {} task to worker #" + (i + 1) ,filesOfWorkers.size()); } } else if ("STOP".equals(message.toString())) { counter ++; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java index c2dba4b..c2b6309 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java @@ -18,53 +18,45 @@ import java.util.List; */ public class EDiffHunkParser extends EDiffParser { - public String testingInfo = ""; - - public int nullMappingGumTreeResult = 0; - public int pureDeletions = 0; - public int largeHunk = 0; - public int nullSourceCode = 0; - public int nullMatchedDiffEntry = 0; - public int testInfos = 0; - public String unfixedViolations = ""; - @Override public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project,String actionType) { List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); if (actionSets.size() != 0) { String folder= null; boolean processActionSet = false; - switch (actionType){ - case "ALL": +// switch (actionType){ +// case "ALL": + if(actionType.equals("ALL")){ folder = "/ALL/"; processActionSet = true; - break; - case "UPD": - processActionSet = - actionSets.stream().allMatch(p -> p.getAction() instanceof Update); + }else if(actionType.equals("UPD") || actionType.equals("INS") || actionType.equals("DEL") || actionType.equals("MOV")|| actionType.equals("MIX")){ + boolean isUPD = actionSets.stream().allMatch(p -> p.getAction() instanceof Update); + boolean isINS = actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); + boolean isDEL = actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); + boolean isMOV = actionSets.stream().allMatch(p -> p.getAction() instanceof Move); + if(isUPD){ folder = "/UPD/"; - break; - case "INS": - processActionSet = - actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); - + processActionSet = true; + }else if(isINS){ folder = "/INS/"; - break; - case "DEL": - processActionSet = - actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); + processActionSet = true; + }else if(isDEL){ folder = "/DEL/"; - break; - case "MOV": - processActionSet = - actionSets.stream().allMatch(p -> p.getAction() instanceof Move); + processActionSet = true; + }else if(isMOV){ folder = "/MOV/"; - break; - default: + processActionSet = true; + }else{ + folder = "/MIX/"; + processActionSet = true; + } + }else{ + + processActionSet = false; System.err.print(actionType + "not known"); - break; + } @@ -78,7 +70,6 @@ public class EDiffHunkParser extends EDiffParser { FileOutputStream f = null; try { -// String pj = diffentryFile.getParent().split("Defects4J")[1]; String datasetName = project; String[] split1 = diffentryFile.getParent().split(datasetName); String root = split1[0]; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java index 00fb5f0..77a30b0 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java @@ -38,27 +38,10 @@ public class EDiffWorker extends UntypedActor { if (message instanceof EDiffMessage) { EDiffMessage msg = (EDiffMessage) message; List files = msg.getMsgFiles(); - StringBuilder editScripts = new StringBuilder(); - StringBuilder patchesSourceCode = new StringBuilder(); - StringBuilder sizes = new StringBuilder(); - StringBuilder tokens = new StringBuilder(); - StringBuilder testingInfo = new StringBuilder(); int id = msg.getId(); int counter = 0; - int nullGumTreeResults = 0; - int noSourceCodeChanges = 0; - int noStatementChanges = 0; - int nullDiffEntry = 0; - int nullMappingGumTreeResults = 0; - int pureDeletion = 0; - int largeHunk = 0; - int nullSourceCode = 0; - int testInfos = 0; - int timeouts = 0; - StringBuilder builder = new StringBuilder(); - for (MessageFile msgFile : files) { File revFile = msgFile.getRevFile(); File prevFile = msgFile.getPrevFile(); @@ -67,41 +50,17 @@ public class EDiffWorker extends UntypedActor { EDiffHunkParser parser = new EDiffHunkParser(); - - final ExecutorService executor = Executors.newWorkStealingPool(); + + final ExecutorService executor = Executors.newSingleThreadExecutor(); // schedule the work final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getActionType())); try { // wait for task to complete future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS); - nullDiffEntry += parser.nullMatchedDiffEntry; - nullMappingGumTreeResults += parser.nullMappingGumTreeResult; - pureDeletion += parser.pureDeletions; - largeHunk += parser.largeHunk; - nullSourceCode += parser.nullSourceCode; - testInfos += parser.testInfos; - testingInfo.append(parser.testingInfo); - builder.append(parser.unfixedViolations); - - String editScript = parser.getAstEditScripts(); - if ("".equals(editScript)) { - - } else { - editScripts.append(editScript); - patchesSourceCode.append(parser.getPatchesSourceCode()); - sizes.append(parser.getSizes()); - tokens.append(parser.getTokensOfSourceCode()); - - counter ++; - if (counter % 100 == 0) { - editScripts.setLength(0); - patchesSourceCode.setLength(0); - sizes.setLength(0); - tokens.setLength(0); - log.info("Worker #" + id +" finalized parsing " + counter + " files..."); - testingInfo.setLength(0); - } + counter ++; + if (counter % 10 == 0) { + log.info("Worker #" + id +" finalized parsing " + counter + " files... remaing "+ (files.size() - counter)); } } catch (TimeoutException e) { future.cancel(true); @@ -116,17 +75,7 @@ public class EDiffWorker extends UntypedActor { executor.shutdownNow(); } } - - if (sizes.length() > 0) { - editScripts.setLength(0); - patchesSourceCode.setLength(0); - sizes.setLength(0); - tokens.setLength(0); - - testingInfo.setLength(0); - } -// log.info("Worker #" + id +" finalized parsing " + counter + " files..."); log.info("Worker #" + id + " finalized the work..."); this.getSender().tell("STOP", getSelf()); } else { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java index ecbdf8d..e154673 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java @@ -24,7 +24,7 @@ public class AkkaTreeLoader { private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class); - public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, int cursor,String chunk,String eDiffTimeout) throws Exception { + public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, int cursor, String chunk, String eDiffTimeout, String parallelism) throws Exception { String parameters = String.format("\nportInner %s \nchunkName %s \ndbDir %s \ndumpsName %s",portInner,chunkName,dbDir,dumpsName); @@ -90,17 +90,17 @@ public class AkkaTreeLoader { - akkaCompare(innerPool,outerPool,numOfWorkers,cursor,eDiffTimeout); + akkaCompare(innerPool,outerPool,numOfWorkers,cursor,eDiffTimeout,parallelism); - String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; - String stopServer2 = String.format(stopServer,Integer.valueOf(port)); - - cs.runShell(stopServer2,port); - - - String stopServer1 = String.format(stopServer,Integer.valueOf(portInner)); - - cs.runShell(stopServer1,portInner); +// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; +// String stopServer2 = String.format(stopServer,Integer.valueOf(port)); +// +// cs.runShell(stopServer2,port); +// +// +// String stopServer1 = String.format(stopServer,Integer.valueOf(portInner)); +// +// cs.runShell(stopServer1,portInner); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java index febe383..afe421b 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java @@ -3,6 +3,7 @@ package edu.lu.uni.serval.fixminer.jobs; import akka.actor.ActorRef; import akka.actor.ActorSystem; import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffHunkParser; import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage; import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile; import edu.lu.uni.serval.utils.FileHelper; @@ -20,7 +21,7 @@ public class EnhancedASTDiff { private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class); - public static void main(String inputPath, String outputPath, String numOfWorkers, String project, String eDiffTimeout, String actionType) { + public static void main(String inputPath, String outputPath, String numOfWorkers, String project, String eDiffTimeout, String actionType, String parallelism) { String parameters = String.format("\nInput path %s \nOutput path %s",inputPath,outputPath); @@ -34,43 +35,81 @@ public class EnhancedASTDiff { .collect(Collectors.toList()); + List allMessageFiles = new ArrayList<>(); for (File target : folders) { - String pjName = target.getName(); + String pjName = target.getName(); - final List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" - System.out.println(msgFiles.size()); - if(msgFiles.size() == 0) - continue; + List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" + allMessageFiles.addAll(msgFiles); +// System.out.println(msgFiles.size()); + if (msgFiles.size() == 0) + continue; - String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; + String GUM_TREE_OUTPUT = outputPath + "/" + pjName + "/"; // a -// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD"); -// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); -// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); -// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/"+actionType); - - int a = 0; - - ActorSystem system = null; - ActorRef parsingActor = null; - final EDiffMessage msg = new EDiffMessage(0, msgFiles,eDiffTimeout,actionType); - try { - log.info("Akka begins..."); - system = ActorSystem.create("Mining-FixPattern-System"); - - parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor"); - parsingActor.tell(msg, ActorRef.noSender()); - } catch (Exception e) { - system.shutdown(); - e.printStackTrace(); + if (actionType.equals("ALL")) { + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/" + actionType); + } else { + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MIX"); } + } + + switch (parallelism){ + case "AKKA": + ActorSystem system = null; + ActorRef parsingActor = null; + final EDiffMessage msg = new EDiffMessage(0, allMessageFiles,eDiffTimeout,actionType); + try { + log.info("Akka begins..."); + log.info("{} files to process ...", allMessageFiles.size()); + system = ActorSystem.create("Mining-FixPattern-System"); + + parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor"); + parsingActor.tell(msg, ActorRef.noSender()); + } catch (Exception e) { + system.shutdown(); + e.printStackTrace(); + } + break; + case "FORKJOIN": + int counter = new Object() { + int counter = 0; + + { + allMessageFiles.stream(). + parallel(). + peek(x -> counter++). + forEach(m -> + { + EDiffHunkParser parser = new EDiffHunkParser(); + parser.parseFixPatterns(m.getPrevFile(),m.getRevFile(), m.getDiffEntryFile(),project,actionType); + if (counter % 10 == 0) { + log.info("Finalized parsing " + counter + " files... remaing " + (allMessageFiles.size() - counter)); + } + } + ); + } + }.counter; + log.info("Finished parsing {} files",counter); + break; + default: + log.error("Unknown parallelism {}", parallelism); + break; + } + + + + } - } + private static List getMessageFiles(String gumTreeInput) { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java index 79bc574..e3a04c7 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java @@ -17,18 +17,21 @@ import java.util.stream.Stream; public class ImportPairs2DB { private static Logger log = LoggerFactory.getLogger(ImportPairs2DB.class); - public static void main(String csvInputPath,String portInner,String dbDir,String datasetPath) throws Exception { + public static void main(String csvInputPath, String portInner, String dbDir, String datasetPath, String chunk) throws Exception { String parameters = String.format("\nInput path %s \nportInner %s \ndbDir %s",csvInputPath,portInner,dbDir); log.info(parameters); + String[] splits = chunk.split("\\."); + String chunkType = splits[splits.length-1]; + log.info("Chunk type {}",chunkType); File folder = new File(csvInputPath); File[] subFolders = folder.listFiles(); Stream stream = Arrays.stream(subFolders); List pjs = stream - .filter(x -> x.getName().endsWith(".txt")) + .filter(x -> x.getName().endsWith(chunkType)) .collect(Collectors.toList()); Integer portInt = Integer.valueOf(portInner); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java index 3feed61..9b993ad 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java @@ -7,7 +7,10 @@ import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; -import edu.lu.uni.serval.utils.*; +import edu.lu.uni.serval.utils.CallShell; +import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.javatuples.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -145,7 +148,7 @@ public class MultiThreadTreeLoaderCluster { ITree parent = null; ITree children =null; TreeContext tc = new TreeContext(); - tree = EDiff.getActionTree(actionSet, parent, children,tc); + tree = EDiffHelper.getActionTree(actionSet, parent, children,tc); tree.setParent(null); tc.validate(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java index 880189d..e542cc5 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java @@ -413,7 +413,7 @@ public class MultiThreadTreeLoaderCluster3 { } // if (oldTokens.size() == 0 ) {// && (oldTree.getType() != 41 && oldTree.getType() != 21 && oldTree.getType() !=17 && oldTree.getType()!=60 && oldTree.getType() != 46)){ -// log.info("dur bakalim nereye!???"); +// log.info("!???"); // } return oldTokens; } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java index 1386f94..7c7d3bc 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java @@ -2,19 +2,18 @@ package edu.lu.uni.serval.fixminer.jobs; import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import edu.lu.uni.serval.utils.Checker; +import edu.lu.uni.serval.utils.EDiffHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; -import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; -import java.util.Base64; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -65,36 +64,9 @@ public class PatternExtractor { public static void loadDB(String inputPath,String portInner,String dbDir,String chunkName,String operation,List fixes) throws Exception { -// String inputPath; -// String portInner; -// String serverWait; -// String dbDir; -// String chunkName; -// String numOfWorkers; -// if (args.length > 0) { -// inputPath = args[0]; -// portInner = args[1]; -// serverWait = args[2]; -// chunkName = args[3]; -// numOfWorkers = args[4]; -// dbDir = args[5]; -// } else { -// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; -// portInner = "6399"; -// serverWait = "10000"; -// chunkName ="dumps.rdb"; -// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; -// numOfWorkers = "1"; -// } -// String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s \noperation %s", inputPath, portInner, serverWait, chunkName, dbDir, operation); -// log.info(parameters); -// CallShell cs = new CallShell(); -// String cmd = "bash " + dbDir + "/" + "startServer.sh" + " %s %s %s"; -// cmd = String.format(cmd, dbDir, chunkName, Integer.valueOf(portInner)); -//// loadRedis(cmd,serverWait); -// cs.runShell(cmd, serverWait); - JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(6399),20000000); + + JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(6399),20000000); @@ -134,11 +106,11 @@ public class PatternExtractor { s = inner.get(fn); if (s != null) { - HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s); BufferedWriter writer = new BufferedWriter(new FileWriter(savePath+saveFN)); - writer.write(toString(actionSet)); + writer.write(EDiffHelper.toString(actionSet)); writer.close(); @@ -207,7 +179,7 @@ public class PatternExtractor { String saveFN = pattern.getName() + "_" + project + "_" + operation + "_" + joinFN; try{ String content = new String(Files.readAllBytes(Paths.get(savePath + saveFN))); - HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(content); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(content); int astType = actionSet.getNode().getType(); if (Checker.isStatement(astType) || astType == 23 //FieldDeclaration @@ -264,41 +236,7 @@ public class PatternExtractor { - static final JedisPoolConfig poolConfig = buildPoolConfig(); - private static String toString( Serializable o ) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream( baos ); - oos.writeObject( o ); - oos.close(); - return Base64.getEncoder().encodeToString(baos.toByteArray()); - } - - /** Read the object from Base64 string. */ - private static Object fromString( String s ) throws IOException , - ClassNotFoundException { - byte[] data = Base64.getDecoder().decode(s); - ObjectInputStream ois = new ObjectInputStream( - new ByteArrayInputStream(data)); - Object o = ois.readObject(); - ois.close(); - return o; - } - private static JedisPoolConfig buildPoolConfig() { - final JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(128); - poolConfig.setMaxIdle(128); - poolConfig.setMinIdle(16); - poolConfig.setTestOnBorrow(false); - poolConfig.setTestOnReturn(false); - poolConfig.setTestWhileIdle(true); - poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); - poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); -// poolConfig.setNumTestsPerEvictionRun(3); - poolConfig.setBlockWhenExhausted(true); - - return poolConfig; - } public static List readIdList(String fileName) { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java index 4acb113..2fbb676 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java @@ -48,24 +48,24 @@ public class StoreEDiffInCache { File[] files = pj.listFiles(); Stream fileStream = Arrays.stream(files); List fs; - if (operation.equals("ALLOP")){ - fs= fileStream - .filter(x -> x.getName().startsWith("UPD") || - x.getName().startsWith("INS") || - x.getName().startsWith("DEL") || - x.getName().startsWith("MOV")) - .collect(Collectors.toList()); - File[] files1 = fs.get(0).listFiles(); - File[] files2 = fs.get(1).listFiles(); - File[] files3 = fs.get(2).listFiles(); - File[] files4 = fs.get(3).listFiles(); - dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new); - }else{ +// if (operation.equals("ALLOP")){ +// fs= fileStream +// .filter(x -> x.getName().startsWith("UPD") || +// x.getName().startsWith("INS") || +// x.getName().startsWith("DEL") || +// x.getName().startsWith("MOV")) +// .collect(Collectors.toList()); +// File[] files1 = fs.get(0).listFiles(); +// File[] files2 = fs.get(1).listFiles(); +// File[] files3 = fs.get(2).listFiles(); +// File[] files4 = fs.get(3).listFiles(); +// dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new); +// }else{ fs = fileStream .filter(x -> x.getName().startsWith(operation)) .collect(Collectors.toList()); dumps = fs.get(0).listFiles(); - } +// } for (File f : dumps) { @@ -115,7 +115,7 @@ public class StoreEDiffInCache { log.error("File not found {}" , path); e.printStackTrace(); } catch (IOException e) { - log.error("Error initializing stream {}" , path); + log.error("Error initializing stream {}", path); e.printStackTrace(); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block diff --git a/src/main/java/edu/lu/uni/serval/utils/EDiff.java b/src/main/java/edu/lu/uni/serval/utils/EDiff.java deleted file mode 100644 index 01fd931..0000000 --- a/src/main/java/edu/lu/uni/serval/utils/EDiff.java +++ /dev/null @@ -1,98 +0,0 @@ -package edu.lu.uni.serval.utils; - -import com.github.gumtreediff.actions.model.*; -import com.github.gumtreediff.tree.ITree; -import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; - -import java.io.IOException; -import java.util.List; - -/** - * Created by anilkoyuncu on 17/09/2018. - */ -public class EDiff { - private static Logger log = LoggerFactory.getLogger(EDiff.class); - public static ITree getSimpliedTree(String fn, JedisPool outerPool) { - - ITree tree = null; - Jedis inner = null; - try { - inner = outerPool.getResource(); - while (!inner.ping().equals("PONG")){ - log.info("wait"); - } - inner.select(1); - String dist2load = inner.get(fn); - inner.select(0); - String s = inner.get(dist2load); - HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s); - - ITree parent = null; - ITree children =null; - TreeContext tc = new TreeContext(); - tree = EDiffHelper.getASTTree(actionSet, parent, children,tc); - tree.setParent(null); - tc.validate(); - - } catch (IOException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - }finally { - if (inner != null) { - inner.close(); - } - } - return tree; - - } - public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){ - - int newType = 0; - - Action action = actionSet.getAction(); - if (action instanceof Update){ - newType = 101; - }else if(action instanceof Insert){ - newType =100; - }else if(action instanceof Move){ - newType = 102; - }else if(action instanceof Delete){ - newType=103; - }else{ - new Exception("unknow action"); - } - if(actionSet.getParent() == null){ - //root - - parent = tc.createTree(newType, "", null); - tc.setRoot(parent); - -// parent = new Tree(newType,""); - }else{ - children = tc.createTree(newType, "", null); - children.setParentAndUpdateChildren(parent); -// children = new Tree(newType,""); -// parent.addChild(children); - } - List subActions = actionSet.getSubActions(); - if (subActions.size() != 0){ - for (HierarchicalActionSet subAction : subActions) { - - if(actionSet.getParent() == null){ - children = parent; - } - getActionTree(subAction,children,null,tc); - - } - - - } - return parent; - } -} diff --git a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java index 74cf247..10fddfe 100644 --- a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java +++ b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java @@ -1,10 +1,13 @@ package edu.lu.uni.serval.utils; +import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; import java.io.*; import java.util.Base64; @@ -126,4 +129,81 @@ public class EDiffHelper { .map(Map.Entry::getKey) .collect(Collectors.toList()); } + + public static ITree getSimpliedTree(String fn, JedisPool outerPool) { + + ITree tree = null; + Jedis inner = null; + try { + inner = outerPool.getResource(); + while (!inner.ping().equals("PONG")){ + log.info("wait"); + } + inner.select(1); + String dist2load = inner.get(fn); + inner.select(0); + String s = inner.get(dist2load); + HierarchicalActionSet actionSet = (HierarchicalActionSet) EDiffHelper.fromString(s); + + ITree parent = null; + ITree children =null; + TreeContext tc = new TreeContext(); + tree = EDiffHelper.getASTTree(actionSet, parent, children,tc); + tree.setParent(null); + tc.validate(); + + } catch (IOException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + }finally { + if (inner != null) { + inner.close(); + } + } + return tree; + + } + public static ITree getActionTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){ + + int newType = 0; + + Action action = actionSet.getAction(); + if (action instanceof Update){ + newType = 101; + }else if(action instanceof Insert){ + newType =100; + }else if(action instanceof Move){ + newType = 102; + }else if(action instanceof Delete){ + newType=103; + }else{ + new Exception("unknow action"); + } + if(actionSet.getParent() == null){ + //root + + parent = tc.createTree(newType, "", null); + tc.setRoot(parent); + + }else{ + children = tc.createTree(newType, "", null); + children.setParentAndUpdateChildren(parent); + + } + List subActions = actionSet.getSubActions(); + if (subActions.size() != 0){ + for (HierarchicalActionSet subAction : subActions) { + + if(actionSet.getParent() == null){ + children = parent; + } + getActionTree(subAction,children,null,tc); + + } + + + } + return parent; + } }