diff --git a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java index 9c8a4f8..2dec952 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/Launcher.java @@ -27,34 +27,35 @@ public class Launcher { appProps.load(new FileInputStream(appConfigPath)); String portInner = appProps.getProperty("portInner","6380"); -// String serverWait = appProps.getProperty("serverWait", "50000"); String numOfWorkers = appProps.getProperty("numOfWorkers", "10"); String jobType = appProps.getProperty("jobType","ALL"); String portDumps = appProps.getProperty("portDumps","6399"); String pythonPath = appProps.getProperty("pythonPath","/Users/anilkoyuncu/bugStudy/code/python"); String datasetPath = appProps.getProperty("datasetPath","/Users/anilkoyuncu/bugStudy/dataset"); String pjName = appProps.getProperty("pjName","allDataset"); -// String dbNo = appProps.getProperty("dbNo","0"); String actionType = appProps.getProperty("actionType","ALL"); String threshold = appProps.getProperty("threshold","1"); String cursor = appProps.getProperty("cursor","10000000"); String chunk = appProps.getProperty("chunk","1.txt"); + String eDiffTimeout = appProps.getProperty("eDiffTimeout","900"); + String isBig = appProps.getProperty("isBigPair","true"); + boolean isBigPair = Boolean.parseBoolean(isBig); String parameters = String.format("\nportInner %s " + "\nnumOfWorkers %s " + "\njobType %s \nport %s " + "\npythonPath %s \ndatasetPath %s" + - "\npjName %s \nactionType %s \nthreshold %s \ncursor %s" - , portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor); + "\npjName %s \nactionType %s \nthreshold %s \ncursor %s \neDiffTimeout %s \nisBigPair %s" + , portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,eDiffTimeout,isBig); log.info(parameters); - mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk); + mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk,eDiffTimeout,isBigPair); } - public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk){ + public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk, String eDiffTimeout, boolean isBigPair){ String dbDir; @@ -68,33 +69,35 @@ public class Launcher { dbDir = datasetPath + "/redis"; pairsPath = datasetPath + "/pairsImport"+pjName; dumpsName = "dumps-"+pjName+".rdb"; + int iCursor = Integer.valueOf(cursor); try { switch (jobType) { case "ENHANCEDASTDIFF": - EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName); + EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName, eDiffTimeout,actionType); break; case "CACHE": StoreEDiffInCache.main(gumOutput, portDumps, dbDir, actionType+dumpsName,actionType); break; case "SI": - CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType); + CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType,isBigPair,iCursor); ImportPairs2DB.main(pairsPath+actionType, portInner, dbDir,datasetPath); break; case "SIMI": - AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,cursor,chunk); + AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,iCursor,chunk,eDiffTimeout); break; case "LEVEL1": - level1(portInner, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput); + String indexFile = pairsPath+actionType + "/" +pjName+actionType+".index"; + level1(portInner, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, gumInput,iCursor,indexFile); break; //CALC python abstractPatch.py to from cluster folder case "LEVEL2": - level2(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); + level2(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput,iCursor); break; //CALC via python case "LEVEL3": - level3(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); + level3(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput,iCursor); break; case "EXTRACTPATTERN": PatternExtractor.mainLaunch(portInner,numOfWorkers,jobType,portDumps,pythonPath,datasetPath,pjName,actionType,threshold); @@ -112,17 +115,17 @@ public class Launcher { } - private static void level1(String portInner, String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String pairsPath, String dumpsName, String gumInput) throws Exception { + private static void level1(String portInner, String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String gumInput,int cursor,String indexFile) throws Exception { - TreeLoaderClusterL1.main(portInner, port, dbDir, "level1-"+pjName+ actionType+".rdb", dbDir ,pjName + actionType); + TreeLoaderClusterL1.main(portInner, port, dbDir, "level1-"+pjName+ actionType+".rdb", dbDir ,pjName + actionType,cursor); CallShell cs1 =new CallShell(); String db1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String db11 = String.format(db1, dbDir,"level1-"+pjName+ actionType+".rdb" ,Integer.valueOf(port)); cs1.runShell(db11, port); - String runpy = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s"; - String formatRunPy = String.format(runpy,pythonPath +"/abstractPatch.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, "matches" + pjName + actionType, threshold); + String runpy = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s %s"; + String formatRunPy = String.format(runpy,pythonPath +"/abstractPatch.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, "matches" + pjName + actionType, threshold,indexFile); cs1.runShell(formatRunPy); String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; @@ -130,18 +133,18 @@ public class Launcher { cs1.runShell(stopServer, port); } - private static void level2(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput) throws Exception { + private static void level2(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput,int cursor) throws Exception { String stopServer; MultiThreadTreeLoaderCluster.calculatePairsOfClusters(datasetPath + "/cluster"+pjName+ actionType, datasetPath,actionType); - MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath+"/pairs"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType); + MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath+"/pairs"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType,cursor); CallShell cs3 =new CallShell(); String db22 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String db1b = String.format(db22, dbDir,"clusterl1-"+pjName+actionType+".rdb",Integer.valueOf(port)); cs3.runShell(db1b, port); - String runpy2 = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s"; - String formatRunPy1a = String.format(runpy2,pythonPath +"/abstractPatchCluster.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold); + String runpy2 = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s %s"; + String formatRunPy1a = String.format(runpy2,pythonPath +"/abstractPatchCluster.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold,"dummy"); cs3.runShell(formatRunPy1a); String stopServer1a = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; @@ -149,18 +152,18 @@ public class Launcher { cs3.runShell(stopServer, port); } - private static void level3(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput) throws Exception { + private static void level3(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput,int cursor) throws Exception { String stopServer; MultiThreadTreeLoaderCluster3.calculatePairsOfClusters(datasetPath + "/cluster-2l"+pjName+ actionType, datasetPath,actionType); - MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath+"/pairs-2l"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType); + MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath+"/pairs-2l"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType,cursor); CallShell cs5 =new CallShell(); String dba = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String dbaa = String.format(dba, dbDir,"clusterl2-"+pjName+actionType+".rdb",Integer.valueOf(port)); cs5.runShell(dbaa, port); - String runpya = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s"; - String formatRunPya = String.format(runpya,pythonPath +"/abstractPatchClusterLevel3.py", gumInput, datasetPath + "/cluster-3l"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold); + String runpya = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s %s"; + String formatRunPya = String.format(runpya,pythonPath +"/abstractPatchClusterLevel3.py", gumInput, datasetPath + "/cluster-3l"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold,"dummy"); cs5.runShell(formatRunPya); String stopServera = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java index 67fed8f..8734393 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java @@ -6,8 +6,8 @@ import com.github.gumtreediff.matchers.Matcher; import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.Tree; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import edu.lu.uni.serval.utils.ASTNodeMap; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.*; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java index be93c0b..f116deb 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java @@ -2,9 +2,9 @@ package edu.lu.uni.serval.fixminer; import akka.actor.ActorRef; import akka.actor.ActorSystem; -import edu.lu.uni.serval.fixminer.cluster.akka.MessageFile; -import edu.lu.uni.serval.fixminer.cluster.akka.EDiffActor; -import edu.lu.uni.serval.fixminer.cluster.akka.EDiffMessage; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage; +import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile; import edu.lu.uni.serval.utils.FileHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,155 +19,155 @@ import java.util.stream.Stream; public class TestHunkParserSingleFile { private static Logger log = LoggerFactory.getLogger(TestHunkParserSingleFile.class); - public static void main(String[] args) { - // input data - -// String rootPath = "/Users/anilkoyuncu/bugStudy"; - String inputPath; - String outputPath; - if(args.length > 0){ - inputPath = args[1]; - outputPath = args[0]; - }else{ -// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeInputBug4"; - inputPath = "/Users/anilkoyuncu/bugStudy/dataset/allDataset"; -// outputPath = "/Users/anilkoyuncu/bugStudy/code/python/GumTreeOutput2/"; - outputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputSingle"; - } - - //5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.txt_1_CAMEL - - File folder = new File(inputPath); - File[] listOfFiles = folder.listFiles(); - Stream stream = Arrays.stream(listOfFiles); - List folders = stream - .filter(x -> !x.getName().startsWith(".")) - - .collect(Collectors.toList()); - -// List targetList = new ArrayList(); -// for (File f:folders){ -// for(File f1 :f.listFiles()){ -// if (!f1.getName().startsWith(".")){ -// targetList.add(f1); -// } -// } +// public static void main(String[] args) { +// // input data +// +//// String rootPath = "/Users/anilkoyuncu/bugStudy"; +// String inputPath; +// String outputPath; +// if(args.length > 0){ +// inputPath = args[1]; +// outputPath = args[0]; +// }else{ +//// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeInputBug4"; +// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/allDataset"; +//// outputPath = "/Users/anilkoyuncu/bugStudy/code/python/GumTreeOutput2/"; +// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputSingle"; // } - -// List pjList = Arrays.asList("DATAJPA","ZXing","PDE","SWS","SWT", "SWF", "COLLECTIONS", "JDT"); - List files = new ArrayList(); -// files.add("5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.java"); - files.add("d6c5e5_9f96d9_hbase-server#src#main#java#org#apache#hadoop#hbase#master#RegionStates.java"); - for(String f : files){ - String pjName = "HBASE"; -// for (File target : folders) { -// String pjName = target.getName(); -// if (!pjList.contains(pjName)){ -// continue; -// } - -// final List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" - MessageFile messageFile = getMessageFile(inputPath + "/" + pjName +"/", f); - - List msgFiles = new ArrayList<>(); - msgFiles.add(messageFile); - String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; - final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list"; - final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list"; - final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list"; - final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv"; - final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list"; - - - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ActionSetDumps"); - FileHelper.deleteDirectory(editScriptsFilePath); - FileHelper.deleteDirectory(patchesSourceCodeFilePath); - FileHelper.deleteDirectory(buggyTokensFilePath); - FileHelper.deleteDirectory(editScriptSizesFilePath); - FileHelper.deleteDirectory(alarmTypesFilePath); - - StringBuilder astEditScripts = new StringBuilder(); - StringBuilder tokens = new StringBuilder(); - StringBuilder sizes = new StringBuilder(); - StringBuilder patches = new StringBuilder(); - StringBuilder alarmTypes = new StringBuilder(); - -// int a = 0; - - ActorSystem system = null; - ActorRef parsingActor = null; - final EDiffMessage msg = new EDiffMessage(0, msgFiles); - try { - log.info("Akka begins..."); - system = ActorSystem.create("Mining-FixPattern-System"); - parsingActor = system.actorOf(EDiffActor.props(1, "dataset"), "mine-fix-pattern-actor"); - parsingActor.tell(msg, ActorRef.noSender()); - } catch (Exception e) { - system.shutdown(); - e.printStackTrace(); - } - -// int counter = 0; -// for (MessageFile msgFile : msgFiles) { -// FixedPatternHunkParser parser = new FixedPatternHunkParser(); // -// final ExecutorService executor = Executors.newSingleThreadExecutor(); -// // schedule the work -// final Future future = executor.submit(new RunnableParser(msgFile.getPrevFile(), -// msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser)); -// try { -// // where we wait for task to complete -// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); -// String editScripts = parser.getAstEditScripts(); -// if (!editScripts.equals("")) { -// astEditScripts.append(editScripts); -// tokens.append(parser.getTokensOfSourceCode()); -// sizes.append(parser.getSizes()); -// patches.append(parser.getPatchesSourceCode()); -// alarmTypes.append(parser.getAlarmTypes()); +// //5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.txt_1_CAMEL // -// a++; -// if (a % 100 == 0) { -// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); -// FileHelper.outputToFile(buggyTokensFilePath, tokens, true); -// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); -// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); -// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); -// astEditScripts.setLength(0); -// tokens.setLength(0); -// sizes.setLength(0); -// patches.setLength(0); -// alarmTypes.setLength(0); -// System.out.println("Finish of parsing " + a + " files......"); -// } -// } -// } catch (TimeoutException e) { -// err.println("task timed out"); -// future.cancel(true /* mayInterruptIfRunning */); -// } catch (InterruptedException e) { -// err.println("task interrupted"); -// } catch (ExecutionException e) { -// err.println("task aborted"); -// } finally { -// executor.shutdownNow(); -// } -// } - - FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); - FileHelper.outputToFile(buggyTokensFilePath, tokens, true); - FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); - FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); - FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); - astEditScripts.setLength(0); - tokens.setLength(0); - sizes.setLength(0); - patches.setLength(0); - alarmTypes.setLength(0); - - -// classifyByAlarmTypes(); - } - } +// File folder = new File(inputPath); +// File[] listOfFiles = folder.listFiles(); +// Stream stream = Arrays.stream(listOfFiles); +// List folders = stream +// .filter(x -> !x.getName().startsWith(".")) +// +// .collect(Collectors.toList()); +// +//// List targetList = new ArrayList(); +//// for (File f:folders){ +//// for(File f1 :f.listFiles()){ +//// if (!f1.getName().startsWith(".")){ +//// targetList.add(f1); +//// } +//// } +//// } +// +//// List pjList = Arrays.asList("DATAJPA","ZXing","PDE","SWS","SWT", "SWF", "COLLECTIONS", "JDT"); +// List files = new ArrayList(); +//// files.add("5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.java"); +// files.add("d6c5e5_9f96d9_hbase-server#src#main#java#org#apache#hadoop#hbase#master#RegionStates.java"); +// for(String f : files){ +// String pjName = "HBASE"; +//// for (File target : folders) { +//// String pjName = target.getName(); +//// if (!pjList.contains(pjName)){ +//// continue; +//// } +// +//// final List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" +// MessageFile messageFile = getMessageFile(inputPath + "/" + pjName +"/", f); +// +// List msgFiles = new ArrayList<>(); +// msgFiles.add(messageFile); +// String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; +// final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list"; +// final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list"; +// final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list"; +// final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv"; +// final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list"; +// +// +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ActionSetDumps"); +// FileHelper.deleteDirectory(editScriptsFilePath); +// FileHelper.deleteDirectory(patchesSourceCodeFilePath); +// FileHelper.deleteDirectory(buggyTokensFilePath); +// FileHelper.deleteDirectory(editScriptSizesFilePath); +// FileHelper.deleteDirectory(alarmTypesFilePath); +// +// StringBuilder astEditScripts = new StringBuilder(); +// StringBuilder tokens = new StringBuilder(); +// StringBuilder sizes = new StringBuilder(); +// StringBuilder patches = new StringBuilder(); +// StringBuilder alarmTypes = new StringBuilder(); +// +//// int a = 0; +// +// ActorSystem system = null; +// ActorRef parsingActor = null; +// final EDiffMessage msg = new EDiffMessage(0, msgFiles); +// try { +// log.info("Akka begins..."); +// system = ActorSystem.create("Mining-FixPattern-System"); +// parsingActor = system.actorOf(EDiffActor.props(1, "dataset"), "mine-fix-pattern-actor"); +// parsingActor.tell(msg, ActorRef.noSender()); +// } catch (Exception e) { +// system.shutdown(); +// e.printStackTrace(); +// } +// +//// int counter = 0; +//// for (MessageFile msgFile : msgFiles) { +//// EDiffHunkParser parser = new EDiffHunkParser(); +//// +//// final ExecutorService executor = Executors.newSingleThreadExecutor(); +//// // schedule the work +//// final Future future = executor.submit(new RunnableParser(msgFile.getPrevFile(), +//// msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser)); +//// try { +//// // where we wait for task to complete +//// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); +//// String editScripts = parser.getAstEditScripts(); +//// if (!editScripts.equals("")) { +//// astEditScripts.append(editScripts); +//// tokens.append(parser.getTokensOfSourceCode()); +//// sizes.append(parser.getSizes()); +//// patches.append(parser.getPatchesSourceCode()); +//// alarmTypes.append(parser.getAlarmTypes()); +//// +//// a++; +//// if (a % 100 == 0) { +//// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); +//// FileHelper.outputToFile(buggyTokensFilePath, tokens, true); +//// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); +//// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); +//// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); +//// astEditScripts.setLength(0); +//// tokens.setLength(0); +//// sizes.setLength(0); +//// patches.setLength(0); +//// alarmTypes.setLength(0); +//// System.out.println("Finish of parsing " + a + " files......"); +//// } +//// } +//// } catch (TimeoutException e) { +//// err.println("task timed out"); +//// future.cancel(true /* mayInterruptIfRunning */); +//// } catch (InterruptedException e) { +//// err.println("task interrupted"); +//// } catch (ExecutionException e) { +//// err.println("task aborted"); +//// } finally { +//// executor.shutdownNow(); +//// } +//// } +// +// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); +// FileHelper.outputToFile(buggyTokensFilePath, tokens, true); +// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); +// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); +// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); +// astEditScripts.setLength(0); +// tokens.setLength(0); +// sizes.setLength(0); +// patches.setLength(0); +// alarmTypes.setLength(0); +// +// +//// classifyByAlarmTypes(); +// } +// } private static List getMessageFiles(String gumTreeInput) { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/BaseMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/BaseMessage.java new file mode 100644 index 0000000..dfdbbde --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/BaseMessage.java @@ -0,0 +1,46 @@ +package edu.lu.uni.serval.fixminer.akka; + +/** + * Created by anilkoyuncu on 18/09/2018. + */ +public class BaseMessage { + + private long SECONDS_TO_WAIT; + private int id; + private int threadPoolSize; + + + public BaseMessage(int id,Long timeout) { + this.id = id; + this.SECONDS_TO_WAIT = timeout; +// this.threadPoolSize = threadPoolSize; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + + + public long getSECONDS_TO_WAIT() { + return SECONDS_TO_WAIT; + } + + public void setSECONDS_TO_WAIT(long SECONDS_TO_WAIT) { + this.SECONDS_TO_WAIT = SECONDS_TO_WAIT; + } + + public int getThreadPoolSize() { + return threadPoolSize; + } + + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } + + +} diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java similarity index 90% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java index c504528..aaa6159 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.compare; import akka.actor.ActorRef; import akka.actor.ActorSystem; @@ -19,7 +19,7 @@ public class AkkaTreeParser { private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class); - public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, String cursor){ + public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, int cursor, String eDiffTimeout){ final List listOfPairs = getMessages(innerPool,cursor); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" @@ -28,7 +28,7 @@ public class AkkaTreeParser { ActorSystem system = null; ActorRef parsingActor = null; - final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool); + final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,eDiffTimeout); try { log.info("Akka begins..."); system = ActorSystem.create("Compare-EnhancedDiff-System"); @@ -41,7 +41,7 @@ public class AkkaTreeParser { } } - public static List getMessages(JedisPool innerPool, String cursor){ + public static List getMessages(JedisPool innerPool, int cursor){ ScanResult scan; @@ -54,7 +54,7 @@ public class AkkaTreeParser { ScanParams sc = new ScanParams(); //150000000 log.info("Scanning "); - sc.count(Integer.valueOf(cursor)); + sc.count(cursor); sc.match("pair_[0-9]*"); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java similarity index 94% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java index cdfcc45..58337b4 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.akka.compare; import com.github.gumtreediff.actions.ActionGenerator; import com.github.gumtreediff.actions.model.Action; @@ -71,7 +71,7 @@ public class Compare { if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j); -// log.info(matchKey); + log.info("{} tagged to be similar" ,matchKey); jedis.select(1); jedis.set(matchKey, result); @@ -86,7 +86,7 @@ public class Compare { } catch (Exception e) { - log.error(e.toString() + " {}", name); + log.debug("{} not comparable", name); // e.printStackTrace(); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/RunnableCompare.java similarity index 90% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/RunnableCompare.java index f51ca09..4ee2ced 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/RunnableCompare.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster; +package edu.lu.uni.serval.fixminer.akka.compare; import redis.clients.jedis.JedisPool; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeActor.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java similarity index 94% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeActor.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java index 6957275..17f660e 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.compare; import akka.actor.ActorRef; import akka.actor.Props; @@ -60,7 +60,7 @@ public class TreeActor extends UntypedActor { int toIndex = (i + 1) * average + counter; List pairsOfWorkers = pairs.subList(fromIndex, toIndex); - final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool); + final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool,((TreeMessage) message).getSECONDS_TO_WAIT()); mineRouter.tell(workMsg, getSelf()); logger.info("Assign a task to worker #" + (i + 1) + "..."); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeMessage.java similarity index 64% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeMessage.java index b23bfe8..a99752c 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeMessage.java @@ -1,5 +1,6 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.compare; +import edu.lu.uni.serval.fixminer.akka.BaseMessage; import redis.clients.jedis.JedisPool; import java.util.List; @@ -7,25 +8,27 @@ import java.util.List; /** * Created by anilkoyuncu on 12/09/2018. */ -public class TreeMessage { - private int id; +public class TreeMessage extends BaseMessage{ private List name; private JedisPool innerPool; private JedisPool outerPool; - public TreeMessage(int id, List name, JedisPool innerPool, JedisPool outerPool) { - this.id = id; + + public TreeMessage(int id, List name, JedisPool innerPool, JedisPool outerPool,String eDiffTimeout) { + super(id,new Long(eDiffTimeout)); + this.name = name; this.innerPool = innerPool; this.outerPool = outerPool; - } - public int getId() { - return id; } + public TreeMessage(int id, List name, JedisPool innerPool, JedisPool outerPool,Long eDiffTimeout) { + super(id,eDiffTimeout); + + this.name = name; + this.innerPool = innerPool; + this.outerPool = outerPool; - public void setId(int id) { - this.id = id; } public List getName() { diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java similarity index 93% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeWorker.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java index 5eb61e1..41849f7 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java @@ -1,11 +1,9 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.compare; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; import edu.lu.uni.serval.config.Configuration; -import edu.lu.uni.serval.fixminer.cluster.Compare; -import edu.lu.uni.serval.fixminer.cluster.RunnableCompare; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.JedisPool; @@ -66,7 +64,7 @@ public class TreeWorker extends UntypedActor { if(message instanceof TreeMessage) { -// if (message instanceof edu.lu.uni.serval.fixminer.cluster.akka.EDiffMessage) { +// if (message instanceof edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage) { TreeMessage msg = (TreeMessage) message; List files = msg.getName(); JedisPool innerPool = msg.getInnerPool(); @@ -92,12 +90,12 @@ public class TreeWorker extends UntypedActor { // }.counter; // - final ExecutorService executor = Executors.newFixedThreadPool(20); + final ExecutorService executor = Executors.newWorkStealingPool(); // // schedule the work final Future future = executor.submit(new RunnableCompare(name, innerPool, outerPool, new Compare())); try { // wait for task to complete - future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); + future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS); Compare compare = new Compare(); compare.coreCompare(name, innerPool, outerPool); counter++; @@ -157,9 +155,8 @@ public class TreeWorker extends UntypedActor { } } - log.info("bitti"); - log.info("Worker #" + id +"finialized parsing " + counter + " files..."); - log.info("Worker #" + id + " finialized the work..."); +// log.info("Worker #" + id +"finialized parsing " + counter + " files..."); + log.info("Worker #" + id + " finalized the work..."); this.getSender().tell("STOP", getSelf()); // String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; // stopServer = String.format(stopServer,Integer.valueOf(innerPort)); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffActor.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java similarity index 88% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffActor.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java index 42be0b0..9a354be 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffActor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.ediff; import akka.actor.ActorRef; import akka.actor.Props; @@ -54,15 +54,15 @@ public class EDiffActor extends UntypedActor { int toIndex = (i + 1) * average + counter; List filesOfWorkers = files.subList(fromIndex, toIndex); - final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers); + final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getActionType()); mineRouter.tell(workMsg, getSelf()); logger.info("Assign a task to worker #" + (i + 1) + "..."); } } else if ("STOP".equals(message.toString())) { counter ++; - logger.info(counter + " workers finailized their work..."); + logger.info(counter + " workers finalized their work..."); if (counter >= numberOfWorkers) { - logger.info("All workers finailized their work..."); + logger.info("All workers finalized their work..."); this.getContext().stop(mineRouter); this.getContext().stop(getSelf()); this.getContext().system().shutdown(); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternHunkParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java similarity index 55% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternHunkParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java index eaf7e7c..c2dba4b 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java @@ -1,4 +1,9 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; + +import com.github.gumtreediff.actions.model.Delete; +import com.github.gumtreediff.actions.model.Insert; +import com.github.gumtreediff.actions.model.Move; +import com.github.gumtreediff.actions.model.Update; import java.io.*; import java.util.List; @@ -11,7 +16,7 @@ import java.util.List; * @author kui.liu * */ -public class FixedPatternHunkParser extends FixedPatternParser { +public class EDiffHunkParser extends EDiffParser { public String testingInfo = ""; @@ -25,30 +30,49 @@ public class FixedPatternHunkParser extends FixedPatternParser { public String unfixedViolations = ""; @Override - public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project) { + public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project,String actionType) { List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); if (actionSets.size() != 0) { -// boolean isUpdate = -// actionSets.stream().allMatch(p -> p.getAction() instanceof Update); -// boolean isInsert = -// actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); -// boolean isDelete = -// actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); -// boolean isMove = -// actionSets.stream().allMatch(p -> p.getAction() instanceof Move); + String folder= null; + boolean processActionSet = false; + switch (actionType){ + case "ALL": + folder = "/ALL/"; + processActionSet = true; + break; + case "UPD": + processActionSet = + actionSets.stream().allMatch(p -> p.getAction() instanceof Update); + folder = "/UPD/"; + break; + case "INS": + processActionSet = + actionSets.stream().allMatch(p -> p.getAction() instanceof Insert); + + folder = "/INS/"; + break; + case "DEL": + processActionSet = + actionSets.stream().allMatch(p -> p.getAction() instanceof Delete); + folder = "/DEL/"; + break; + case "MOV": + processActionSet = + actionSets.stream().allMatch(p -> p.getAction() instanceof Move); + folder = "/MOV/"; + break; + default: + processActionSet = false; + System.err.print(actionType + "not known"); + break; + } + + int hunkSet = 0; -// if (isUpdate || isInsert || isDelete || isMove) { + if(processActionSet){ for (HierarchicalActionSet actionSet : actionSets) { - String folder = "/ALL/"; -// if (isUpdate) { -// folder = "/UPD/"; -// } else if (isDelete) { -// folder = "/DEL/"; -// } else if (isInsert) { -// folder = "/INS/"; -// } else if (isMove) { -// folder = "/MOV/"; -// } + + FileOutputStream f = null; @@ -75,7 +99,7 @@ public class FixedPatternHunkParser extends FixedPatternParser { hunkSet++; } -// } + } } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java new file mode 100644 index 0000000..ac5e11d --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java @@ -0,0 +1,45 @@ +package edu.lu.uni.serval.fixminer.akka.ediff; + +import edu.lu.uni.serval.fixminer.akka.BaseMessage; + +import java.util.List; + +public class EDiffMessage extends BaseMessage{ + + + private List msgFiles; + + + + private String actionType; + + + + + + public EDiffMessage(int id, List msgFiles,String eDiffTimeout,String actionType) { + super(id,new Long(eDiffTimeout)); + this.msgFiles = msgFiles; + this.actionType = actionType; + } + public EDiffMessage(int id, List msgFiles,Long eDiffTimeout,String actionType) { + super(id,eDiffTimeout); + this.msgFiles = msgFiles; + this.actionType = actionType; + } + + public String getActionType() { + return actionType; + } + + public void setActionType(String actionType) { + this.actionType = actionType; + } + + + + public List getMsgFiles() { + return msgFiles; + } + +} \ No newline at end of file diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java similarity index 79% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java index ae3255a..8e0e9d7 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import com.github.gumtreediff.actions.model.Action; import edu.lu.uni.serval.gumtree.GumTreeComparer; @@ -14,7 +14,7 @@ import java.util.List; * @author kui.liu * */ -public class FixedPatternParser extends Parser { +public class EDiffParser extends Parser { /* * ResultType: @@ -26,11 +26,7 @@ public class FixedPatternParser extends Parser { */ public int resultType = 0; - protected String violationTypes = ""; - @Override - public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project) { - } /** * Regroup GumTree results without remove the modification of variable names. @@ -65,16 +61,11 @@ public class FixedPatternParser extends Parser { } } + @Override + public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, String actionType) { + + } - -// @Override -// public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) { -// -// } - -// public void setUselessViolations(List uselessViolations) { -// this.uselessViolations = uselessViolations; -// } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffWorker.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java similarity index 83% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffWorker.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java index 5c8a77f..00fb5f0 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffWorker.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java @@ -1,11 +1,8 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.ediff; import akka.actor.Props; import akka.actor.UntypedActor; import akka.japi.Creator; -import edu.lu.uni.serval.FixPatternParser.FixedPatternHunkParser; -import edu.lu.uni.serval.FixPatternParser.RunnableParser; -import edu.lu.uni.serval.config.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,14 +65,15 @@ public class EDiffWorker extends UntypedActor { File diffentryFile = msgFile.getDiffEntryFile(); - FixedPatternHunkParser parser = new FixedPatternHunkParser(); + + EDiffHunkParser parser = new EDiffHunkParser(); - final ExecutorService executor = Executors.newSingleThreadExecutor(); + final ExecutorService executor = Executors.newWorkStealingPool(); // schedule the work - final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project)); + final Future future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getActionType())); try { // wait for task to complete - future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); + future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS); nullDiffEntry += parser.nullMatchedDiffEntry; nullMappingGumTreeResults += parser.nullMappingGumTreeResult; @@ -101,7 +99,7 @@ public class EDiffWorker extends UntypedActor { patchesSourceCode.setLength(0); sizes.setLength(0); tokens.setLength(0); - log.info("Worker #" + id +" finialized parsing " + counter + " files..."); + log.info("Worker #" + id +" finalized parsing " + counter + " files..."); testingInfo.setLength(0); } } @@ -128,8 +126,8 @@ public class EDiffWorker extends UntypedActor { testingInfo.setLength(0); } - log.info("Worker #" + id +"finialized parsing " + counter + " files..."); - log.info("Worker #" + id + " finialized the work..."); +// log.info("Worker #" + id +" finalized parsing " + counter + " files..."); + log.info("Worker #" + id + " finalized the work..."); this.getSender().tell("STOP", getSelf()); } else { unhandled(message); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalActionSet.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java similarity index 98% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalActionSet.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java index a0125b6..3a019e0 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalActionSet.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import com.github.gumtreediff.actions.model.Action; import com.github.gumtreediff.tree.ITree; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalRegrouper.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java similarity index 99% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalRegrouper.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java index 004c773..5549e2f 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalRegrouper.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.tree.ITree; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/MessageFile.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java similarity index 93% rename from src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/MessageFile.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java index e2bea96..b51e597 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/MessageFile.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; +package edu.lu.uni.serval.fixminer.akka.ediff; import java.io.File; diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java similarity index 92% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java index 2b7367b..21bd9b4 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import java.io.File; @@ -18,7 +18,7 @@ public abstract class Parser implements ParserInterface { protected String originalTree = ""; // Guide of generating patches. protected String actionSets = ""; // Guide of generating patches. - public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile,String project); + public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile,String project,String actionType); @Override diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java similarity index 88% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java index 2a4c517..aac3e0f 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; public interface ParserInterface { diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java similarity index 80% rename from src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java rename to src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java index 891af4f..b74726f 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java @@ -1,4 +1,4 @@ -package edu.lu.uni.serval.FixPatternParser; +package edu.lu.uni.serval.fixminer.akka.ediff; import java.io.File; @@ -9,6 +9,7 @@ public class RunnableParser implements Runnable { private File diffentryFile; private Parser parser; private String project; + private String actionType; public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) { this.prevFile = prevFile; @@ -17,16 +18,17 @@ public class RunnableParser implements Runnable { this.parser = parser; } - public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser,String project) { + public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser,String project,String actionType) { this.prevFile = prevFile; this.revFile = revFile; this.diffentryFile = diffentryFile; this.parser = parser; this.project = project; + this.actionType= actionType; } @Override public void run() { - parser.parseFixPatterns(prevFile, revFile, diffentryFile,project); + parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,actionType); } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffMessage.java b/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffMessage.java deleted file mode 100644 index afea756..0000000 --- a/src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffMessage.java +++ /dev/null @@ -1,24 +0,0 @@ -package edu.lu.uni.serval.fixminer.cluster.akka; - -import java.util.List; - -public class EDiffMessage { - - private int id; - private List msgFiles; - - public EDiffMessage(int id, List msgFiles) { - super(); - this.id = id; - this.msgFiles = msgFiles; - } - - public int getId() { - return id; - } - - public List getMsgFiles() { - return msgFiles; - } - -} \ No newline at end of file diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java index 439c845..ecbdf8d 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java @@ -14,7 +14,7 @@ import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; -import static edu.lu.uni.serval.fixminer.cluster.akka.AkkaTreeParser.akkaCompare; +import static edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser.akkaCompare; /** * Created by anilkoyuncu on 19/03/2018. @@ -24,7 +24,7 @@ public class AkkaTreeLoader { private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class); - public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, String cursor,String chunk) throws Exception { + public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, int cursor,String chunk,String eDiffTimeout) throws Exception { String parameters = String.format("\nportInner %s \nchunkName %s \ndbDir %s \ndumpsName %s",portInner,chunkName,dbDir,dumpsName); @@ -90,7 +90,17 @@ public class AkkaTreeLoader { - akkaCompare(innerPool,outerPool,numOfWorkers,cursor); + akkaCompare(innerPool,outerPool,numOfWorkers,cursor,eDiffTimeout); + + String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; + String stopServer2 = String.format(stopServer,Integer.valueOf(port)); + + cs.runShell(stopServer2,port); + + + String stopServer1 = String.format(stopServer,Integer.valueOf(portInner)); + + cs.runShell(stopServer1,portInner); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java index dfaa3c0..494857d 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java @@ -1,8 +1,8 @@ package edu.lu.uni.serval.fixminer.jobs; import edu.lu.uni.serval.utils.CallShell; -import edu.lu.uni.serval.utils.PoolBuilder; import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -22,7 +22,7 @@ import java.util.List; public class CalculatePairs { private static Logger log = LoggerFactory.getLogger(CalculatePairs.class); - public static void main(String dbDir,String chunkName,String port,String outputPath,String pjName) throws Exception { + public static void main(String dbDir,String chunkName,String port,String outputPath,String pjName,boolean isBigPair,int cursor) throws Exception { String parameters = String.format("\nport %s \nchunkName %s \ndbDir %s",port,chunkName,dbDir); @@ -46,7 +46,7 @@ public class CalculatePairs { ScanParams sc = new ScanParams(); //150000000 - sc.count(1500000000); + sc.count(cursor); sc.match("*"); scan = outer.scan("0", sc); @@ -60,31 +60,59 @@ public class CalculatePairs { byte [] buf = new byte[0]; String line = null; + if(isBigPair) { + bigPair(outputPath, pjName, result, buf, line); + }else{ + smallPair(outputPath, pjName, result, buf, line); + } + +// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; +// String stopServer2 = String.format(stopServer,Integer.valueOf(port)); +//// loadRedis(stopServer2,serverWait); +// cs.runShell(stopServer2,serverWait); + log.info("Done pairs"); + } + + private static void smallPair(String outputPath, String pjName, List result, byte[] buf, String line) { try { -// FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv"); -// DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); -// + FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv"); + DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); + + FileOutputStream fosIndex = new FileOutputStream(outputPath + "/" +pjName+".index"); + DataOutputStream outStreamIndex = new DataOutputStream(new BufferedOutputStream(fosIndex)); + + for (int i = 0; i < result.size(); i++) { + line = String.valueOf(i) +"," + result.get(i)+"\n"; + outStreamIndex.write(line.getBytes()); + + for (int j = i + 1; j < result.size(); j++) { + + + + line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n"; + outStream.write(line.getBytes()); + + } + } + outStream.close(); + outStreamIndex.close(); + + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + }catch (java.nio.BufferOverflowException e) { + log.error(line); + log.error(String.valueOf(buf.length)); + e.printStackTrace(); + } + } + + private static void bigPair(String outputPath, String pjName, List result, byte[] buf, String line) { + try { + FileOutputStream fosIndex = new FileOutputStream(outputPath + "/" +pjName+".index"); DataOutputStream outStreamIndex = new DataOutputStream(new BufferedOutputStream(fosIndex)); -// -// -// -// -// for (int i = 0; i < result.size(); i++) { -// line = String.valueOf(i) +"," + result.get(i)+"\n"; -// outStreamIndex.write(line.getBytes()); -// -// for (int j = i + 1; j < result.size(); j++) { -// -// -// -// line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n"; -// outStream.write(line.getBytes()); -// -// } -// } -// outStream.close(); -// outStreamIndex.close(); int fileCounter = 0; FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +pjName +String.valueOf(fileCounter)+".txt", "rw").getChannel(); int maxSize = 500*1000000; @@ -97,8 +125,6 @@ public class CalculatePairs { for (int j = i + 1; j < result.size(); j++) { - -// line = String.valueOf(i) +"\t" + String.valueOf(j)+"\n"; // + "\t" + result.get(i) + "\t" + result.get(j)+"\n"; line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n"; buf = line.getBytes(); if(wrBuf.remaining() > 500) { @@ -126,12 +152,6 @@ public class CalculatePairs { log.error(String.valueOf(buf.length)); e.printStackTrace(); } - -// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; -// String stopServer2 = String.format(stopServer,Integer.valueOf(port)); -//// loadRedis(stopServer2,serverWait); -// cs.runShell(stopServer2,serverWait); - log.info("Done pairs"); } } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java index 0ec8660..febe383 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java @@ -2,9 +2,9 @@ package edu.lu.uni.serval.fixminer.jobs; import akka.actor.ActorRef; import akka.actor.ActorSystem; -import edu.lu.uni.serval.fixminer.cluster.akka.EDiffMessage; -import edu.lu.uni.serval.fixminer.cluster.akka.MessageFile; -import edu.lu.uni.serval.fixminer.cluster.akka.EDiffActor; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor; +import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage; +import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile; import edu.lu.uni.serval.utils.FileHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,7 +20,7 @@ public class EnhancedASTDiff { private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class); - public static void main(String inputPath, String outputPath,String numOfWorkers,String project) { + public static void main(String inputPath, String outputPath, String numOfWorkers, String project, String eDiffTimeout, String actionType) { String parameters = String.format("\nInput path %s \nOutput path %s",inputPath,outputPath); @@ -45,18 +45,19 @@ public class EnhancedASTDiff { String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); - FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ALL"); +// a +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); +// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/"+actionType); int a = 0; ActorSystem system = null; ActorRef parsingActor = null; - final EDiffMessage msg = new EDiffMessage(0, msgFiles); + final EDiffMessage msg = new EDiffMessage(0, msgFiles,eDiffTimeout,actionType); try { log.info("Akka begins..."); system = ActorSystem.create("Mining-FixPattern-System"); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java index bf2befb..79bc574 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java @@ -44,7 +44,7 @@ public class ImportPairs2DB { cmd = String.format(cmd, pj.getPath(), portInt); log.info(cmd); - cs.runShell(cmd); + cs.runShell(cmd,portInner); String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; String stopServer2 = String.format(stopServer,portInt); diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java index 723babf..3feed61 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java @@ -6,12 +6,8 @@ import com.github.gumtreediff.matchers.Matcher; import com.github.gumtreediff.matchers.Matchers; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.utils.CallShell; -import edu.lu.uni.serval.utils.EDiff; -import edu.lu.uni.serval.utils.EDiffHelper; -import edu.lu.uni.serval.utils.PoolBuilder; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; -import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; +import edu.lu.uni.serval.utils.*; import org.javatuples.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,7 +32,7 @@ public class MultiThreadTreeLoaderCluster { private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoaderCluster.class); - public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type) throws Exception { + public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type,int cursor) throws Exception { CallShell cs = new CallShell(); String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; @@ -74,7 +70,7 @@ public class MultiThreadTreeLoaderCluster { try (Jedis jedis = jedisPool.getResource()) { // do operations with jedis resource ScanParams sc = new ScanParams(); - sc.count(150000000); + sc.count(cursor); sc.match("pair_[0-9]*"); log.info("Scanning"); @@ -84,13 +80,14 @@ public class MultiThreadTreeLoaderCluster { if (size == 0) { String comd = String.format(cmd3,f.getPath(),portInner); // loadRedis(comd); - cs.runShell(comd); + log.info("Importing {} pairs for cluster {}",size,f.getName()); + cs.runShell(comd,portInner); scan = jedis.scan("0", sc); size = scan.getResult().size(); } - log.info("Scanned " + String.valueOf(size)); + log.info("Scanned {} for cluster {}",String.valueOf(size),f.getName()); String clusterName = f.getName().replaceAll("[^0-9]+", ""); @@ -247,7 +244,7 @@ public class MultiThreadTreeLoaderCluster { // log.info("Completed " + resultKey); }catch (Exception e){ - log.warn(e.toString() + " {}",(name)); + log.debug(e.toString() + " {}",(name)); } @@ -329,7 +326,7 @@ public class MultiThreadTreeLoaderCluster { e.printStackTrace(); } - log.info("Done pairs"); + log.info("Done pairs of cluster {}",cluster); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java index 73562b8..880189d 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java @@ -2,11 +2,11 @@ package edu.lu.uni.serval.fixminer.jobs; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import edu.lu.uni.serval.utils.CallShell; import edu.lu.uni.serval.utils.EDiffHelper; -import edu.lu.uni.serval.utils.PoolBuilder; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import edu.lu.uni.serval.utils.FileHelper; +import edu.lu.uni.serval.utils.PoolBuilder; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.similarity.JaroWinklerDistance; import org.slf4j.Logger; @@ -33,7 +33,7 @@ public class MultiThreadTreeLoaderCluster3 { private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoaderCluster3.class); - public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type) throws Exception { + public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type,int cursor) throws Exception { CallShell cs = new CallShell(); String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; @@ -69,7 +69,7 @@ public class MultiThreadTreeLoaderCluster3 { try (Jedis jedis = jedisPool.getResource()) { // do operations with jedis resource ScanParams sc = new ScanParams(); - sc.count(150000000); + sc.count(cursor); sc.match("pair_[0-9]*"); log.info("Scanning"); @@ -78,7 +78,7 @@ public class MultiThreadTreeLoaderCluster3 { if (size == 0) { String comd = String.format(cmd3, f.getPath(), portInner); - cs.runShell(comd); + cs.runShell(comd,portInner); scan = jedis.scan("0", sc); @@ -149,7 +149,7 @@ public class MultiThreadTreeLoaderCluster3 { ITree parent = null; ITree children =null; TreeContext tc = new TreeContext(); - tree = EDiffHelper.getASTTree(actionSet, parent, children,tc); + tree = EDiffHelper.getTokenTree(actionSet, parent, children,tc); // tree.setParent(null); tc.validate(); @@ -526,7 +526,7 @@ public class MultiThreadTreeLoaderCluster3 { .collect(Collectors.toList()); FileHelper.createDirectory(outputPath + "/pairs-2l"+type); - + log.info("Calculating pairs"); for (File pj : pjs) { File[] files = pj.listFiles(); List fileList = Arrays.asList(files); @@ -536,6 +536,7 @@ public class MultiThreadTreeLoaderCluster3 { } File[] clusterFiles = cluster.listFiles(); List clusterFilesL = Arrays.asList(clusterFiles); + readMessageFilesCluster(clusterFilesL, outputPath, inputPath, pj.getName(), cluster.getName(),type); @@ -559,7 +560,7 @@ public class MultiThreadTreeLoaderCluster3 { treesFileNames.add(target.toString()); } - log.info("Calculating pairs"); + // treesFileNames = treesFileNames.subList(0,100); String filename = "cluster" + cluster + "_" + subCluster; @@ -609,7 +610,7 @@ public class MultiThreadTreeLoaderCluster3 { e.printStackTrace(); } - log.info("Done pairs"); + log.info("Done pairs of {}",filename); } diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java index 0dd8b6e..1386f94 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java @@ -1,7 +1,7 @@ package edu.lu.uni.serval.fixminer.jobs; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import edu.lu.uni.serval.utils.Checker; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java index 9e5541d..4acb113 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java @@ -1,9 +1,9 @@ package edu.lu.uni.serval.fixminer.jobs; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import edu.lu.uni.serval.utils.CallShell; import edu.lu.uni.serval.utils.EDiffHelper; import edu.lu.uni.serval.utils.PoolBuilder; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -112,10 +112,10 @@ public class StoreEDiffInCache { } catch (FileNotFoundException e) { - log.error("File not found"); + log.error("File not found {}" , path); e.printStackTrace(); } catch (IOException e) { - log.error("Error initializing stream"); + log.error("Error initializing stream {}" , path); e.printStackTrace(); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block diff --git a/src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java b/src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java index 99a32fe..3c0818f 100644 --- a/src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java +++ b/src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java @@ -22,7 +22,7 @@ public class TreeLoaderClusterL1 { private static Logger log = LoggerFactory.getLogger(TreeLoaderClusterL1.class); - public static void main(String portInner,String port,String inputPath,String level1DB,String level1Path,String innerTypePrefix) throws Exception { + public static void main(String portInner,String port,String inputPath,String level1DB,String level1Path,String innerTypePrefix,int cursor) throws Exception { @@ -59,7 +59,7 @@ public class TreeLoaderClusterL1 { inner.select(1); ScanParams sc = new ScanParams(); //150000000 - sc.count(150000000); + sc.count(cursor); sc.match("match_[0-9]*"); ScanResult scan; scan = inner.scan("0", sc); diff --git a/src/main/java/edu/lu/uni/serval/utils/CallShell.java b/src/main/java/edu/lu/uni/serval/utils/CallShell.java index 9e4b56d..85c76ab 100644 --- a/src/main/java/edu/lu/uni/serval/utils/CallShell.java +++ b/src/main/java/edu/lu/uni/serval/utils/CallShell.java @@ -4,6 +4,9 @@ package edu.lu.uni.serval.utils; * Created by anilkoyuncu on 17/04/2018. */ +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -11,6 +14,7 @@ import java.util.concurrent.TimeUnit; public class CallShell { + private static Logger log = LoggerFactory.getLogger(CallShell.class); @@ -28,6 +32,8 @@ public class CallShell { public static void runShell(String command, String port) throws Exception { + log.info(command); + Process process = Runtime.getRuntime().exec(command); BufferedReader reader = new BufferedReader(new InputStreamReader( process.getInputStream())); diff --git a/src/main/java/edu/lu/uni/serval/utils/EDiff.java b/src/main/java/edu/lu/uni/serval/utils/EDiff.java index a2a1bfc..01fd931 100644 --- a/src/main/java/edu/lu/uni/serval/utils/EDiff.java +++ b/src/main/java/edu/lu/uni/serval/utils/EDiff.java @@ -3,7 +3,7 @@ package edu.lu.uni.serval.utils; import com.github.gumtreediff.actions.model.*; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; diff --git a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java index ae7d3f0..74cf247 100644 --- a/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java +++ b/src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java @@ -2,7 +2,7 @@ package edu.lu.uni.serval.utils; import com.github.gumtreediff.tree.ITree; import com.github.gumtreediff.tree.TreeContext; -import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet; +import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +38,47 @@ public class EDiffHelper { return Base64.getEncoder().encodeToString(baos.toByteArray()); } + + public static ITree getTokenTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){ + + int newType = 0; + + String astNodeType = actionSet.getAstNodeType(); + + String label = actionSet.getAction().toString(); + List keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType); + + if(keysByValue.size() != 1){ + log.error("More than 1"); + } + newType = keysByValue.get(0); + if(actionSet.getParent() == null){ + //root + + parent = tc.createTree(newType, label, null); + tc.setRoot(parent); + }else{ + + children = tc.createTree(newType, label, null); + children.setParentAndUpdateChildren(parent); + } + List subActions = actionSet.getSubActions(); + if (subActions.size() != 0){ + for (HierarchicalActionSet subAction : subActions) { + + if(actionSet.getParent() == null){ + children = parent; + } + getTokenTree(subAction,children,null,tc); + + } + + + } + return parent; + } + + public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children, TreeContext tc){ int newType = 0;