azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rc 4

Browse Source
This commit is contained in:
fixminer
2018-09-18 14:59:44 +02:00
parent 3d20268377
commit b76fd00695
34 changed files with 527 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+27 -24
View File
@@ -27,34 +27,35 @@ public class Launcher {
appProps.load(new FileInputStream(appConfigPath));
String portInner = appProps.getProperty("portInner","6380");
// String serverWait = appProps.getProperty("serverWait", "50000");
String numOfWorkers = appProps.getProperty("numOfWorkers", "10");
String jobType = appProps.getProperty("jobType","ALL");
String portDumps = appProps.getProperty("portDumps","6399");
String pythonPath = appProps.getProperty("pythonPath","/Users/anilkoyuncu/bugStudy/code/python");
String datasetPath = appProps.getProperty("datasetPath","/Users/anilkoyuncu/bugStudy/dataset");
String pjName = appProps.getProperty("pjName","allDataset");
// String dbNo = appProps.getProperty("dbNo","0");
String actionType = appProps.getProperty("actionType","ALL");
String threshold = appProps.getProperty("threshold","1");
String cursor = appProps.getProperty("cursor","10000000");
String chunk = appProps.getProperty("chunk","1.txt");
String eDiffTimeout = appProps.getProperty("eDiffTimeout","900");
String isBig = appProps.getProperty("isBigPair","true");
boolean isBigPair = Boolean.parseBoolean(isBig);
String parameters = String.format("\nportInner %s " +
"\nnumOfWorkers %s " +
"\njobType %s \nport %s " +
"\npythonPath %s \ndatasetPath %s" +
"\npjName %s \nactionType %s \nthreshold %s \ncursor %s"
, portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor);
"\npjName %s \nactionType %s \nthreshold %s \ncursor %s \neDiffTimeout %s \nisBigPair %s"
, portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,eDiffTimeout,isBig);
log.info(parameters);
mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk);
mainLaunch(portInner, numOfWorkers, jobType, portDumps, pythonPath,datasetPath,pjName,actionType,threshold,cursor,chunk,eDiffTimeout,isBigPair);
}
public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk){
public static void mainLaunch(String portInner, String numOfWorkers, String jobType, String portDumps, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String cursor, String chunk, String eDiffTimeout, boolean isBigPair){
String dbDir;
@@ -68,33 +69,35 @@ public class Launcher {
dbDir = datasetPath + "/redis";
pairsPath = datasetPath + "/pairsImport"+pjName;
dumpsName = "dumps-"+pjName+".rdb";
int iCursor = Integer.valueOf(cursor);
try {
switch (jobType) {
case "ENHANCEDASTDIFF":
EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName);
EnhancedASTDiff.main(gumInput, gumOutput, numOfWorkers, pjName, eDiffTimeout,actionType);
break;
case "CACHE":
StoreEDiffInCache.main(gumOutput, portDumps, dbDir, actionType+dumpsName,actionType);
break;
case "SI":
CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType);
CalculatePairs.main(dbDir, actionType+dumpsName, portDumps, pairsPath+actionType, pjName+actionType,isBigPair,iCursor);
ImportPairs2DB.main(pairsPath+actionType, portInner, dbDir,datasetPath);
break;
case "SIMI":
AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,cursor,chunk);
AkkaTreeLoader.main(portInner, dbDir, pjName +actionType+chunk+".rdb" , portDumps, actionType+dumpsName,pairsPath+actionType,numOfWorkers,iCursor,chunk,eDiffTimeout);
break;
case "LEVEL1":
level1(portInner, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput);
String indexFile = pairsPath+actionType + "/" +pjName+actionType+".index";
level1(portInner, portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, gumInput,iCursor,indexFile);
break;
//CALC python abstractPatch.py to from cluster folder
case "LEVEL2":
level2(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
level2(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput,iCursor);
break;
//CALC via python
case "LEVEL3":
level3(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput);
level3(portDumps, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput,iCursor);
break;
case "EXTRACTPATTERN":
PatternExtractor.mainLaunch(portInner,numOfWorkers,jobType,portDumps,pythonPath,datasetPath,pjName,actionType,threshold);
@@ -112,17 +115,17 @@ public class Launcher {
}
private static void level1(String portInner, String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String pairsPath, String dumpsName, String gumInput) throws Exception {
private static void level1(String portInner, String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String gumInput,int cursor,String indexFile) throws Exception {
TreeLoaderClusterL1.main(portInner, port, dbDir, "level1-"+pjName+ actionType+".rdb", dbDir ,pjName + actionType);
TreeLoaderClusterL1.main(portInner, port, dbDir, "level1-"+pjName+ actionType+".rdb", dbDir ,pjName + actionType,cursor);
CallShell cs1 =new CallShell();
String db1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
String db11 = String.format(db1, dbDir,"level1-"+pjName+ actionType+".rdb" ,Integer.valueOf(port));
cs1.runShell(db11, port);
String runpy = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s";
String formatRunPy = String.format(runpy,pythonPath +"/abstractPatch.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, "matches" + pjName + actionType, threshold);
String runpy = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s %s";
String formatRunPy = String.format(runpy,pythonPath +"/abstractPatch.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, "matches" + pjName + actionType, threshold,indexFile);
cs1.runShell(formatRunPy);
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
@@ -130,18 +133,18 @@ public class Launcher {
cs1.runShell(stopServer, port);
}
private static void level2(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput) throws Exception {
private static void level2(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput,int cursor) throws Exception {
String stopServer;
MultiThreadTreeLoaderCluster.calculatePairsOfClusters(datasetPath + "/cluster"+pjName+ actionType, datasetPath,actionType);
MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath+"/pairs"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType);
MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath+"/pairs"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType,cursor);
CallShell cs3 =new CallShell();
String db22 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
String db1b = String.format(db22, dbDir,"clusterl1-"+pjName+actionType+".rdb",Integer.valueOf(port));
cs3.runShell(db1b, port);
String runpy2 = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s";
String formatRunPy1a = String.format(runpy2,pythonPath +"/abstractPatchCluster.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold);
String runpy2 = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s %s";
String formatRunPy1a = String.format(runpy2,pythonPath +"/abstractPatchCluster.py", gumInput, datasetPath + "/cluster"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold,"dummy");
cs3.runShell(formatRunPy1a);
String stopServer1a = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
@@ -149,18 +152,18 @@ public class Launcher {
cs3.runShell(stopServer, port);
}
private static void level3(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput) throws Exception {
private static void level3(String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String dumpsName, String gumInput,int cursor) throws Exception {
String stopServer;
MultiThreadTreeLoaderCluster3.calculatePairsOfClusters(datasetPath + "/cluster-2l"+pjName+ actionType, datasetPath,actionType);
MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath+"/pairs-2l"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType);
MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath+"/pairs-2l"+actionType, datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-"+pjName+actionType+".rdb", actionType+dumpsName, "6301",actionType,cursor);
CallShell cs5 =new CallShell();
String dba = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
String dbaa = String.format(dba, dbDir,"clusterl2-"+pjName+actionType+".rdb",Integer.valueOf(port));
cs5.runShell(dbaa, port);
String runpya = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s";
String formatRunPya = String.format(runpya,pythonPath +"/abstractPatchClusterLevel3.py", gumInput, datasetPath + "/cluster-3l"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold);
String runpya = "bash "+datasetPath + "/" + "launchPy.sh" +" %s %s %s %s %s %s %s";
String formatRunPya = String.format(runpya,pythonPath +"/abstractPatchClusterLevel3.py", gumInput, datasetPath + "/cluster-3l"+pjName+ actionType, port, datasetPath + "/cluster-2l"+pjName+ actionType,threshold,"dummy");
cs5.runShell(formatRunPya);
String stopServera = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
src/main/java/edu/lu/uni/serval/fixminer/MultiThreadTreeLoader.java
+1 -1
View File
@@ -6,8 +6,8 @@ import com.github.gumtreediff.matchers.Matcher;
import com.github.gumtreediff.matchers.Matchers;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.Tree;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.utils.ASTNodeMap;
import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.*;
src/main/java/edu/lu/uni/serval/fixminer/TestHunkParserSingleFile.java
+149 -149
View File
@@ -2,9 +2,9 @@ package edu.lu.uni.serval.fixminer;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import edu.lu.uni.serval.fixminer.cluster.akka.MessageFile;
import edu.lu.uni.serval.fixminer.cluster.akka.EDiffActor;
import edu.lu.uni.serval.fixminer.cluster.akka.EDiffMessage;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage;
import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile;
import edu.lu.uni.serval.utils.FileHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -19,155 +19,155 @@ import java.util.stream.Stream;
public class TestHunkParserSingleFile {
private static Logger log = LoggerFactory.getLogger(TestHunkParserSingleFile.class);
public static void main(String[] args) {
// input data
// String rootPath = "/Users/anilkoyuncu/bugStudy";
String inputPath;
String outputPath;
if(args.length > 0){
inputPath = args[1];
outputPath = args[0];
}else{
// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeInputBug4";
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/allDataset";
// outputPath = "/Users/anilkoyuncu/bugStudy/code/python/GumTreeOutput2/";
outputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputSingle";
}
//5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.txt_1_CAMEL
File folder = new File(inputPath);
File[] listOfFiles = folder.listFiles();
Stream<File> stream = Arrays.stream(listOfFiles);
List<File> folders = stream
.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
// List<File> targetList = new ArrayList<File>();
// for (File f:folders){
// for(File f1 :f.listFiles()){
// if (!f1.getName().startsWith(".")){
// targetList.add(f1);
// }
// }
// public static void main(String[] args) {
// // input data
//
//// String rootPath = "/Users/anilkoyuncu/bugStudy";
// String inputPath;
// String outputPath;
// if(args.length > 0){
// inputPath = args[1];
// outputPath = args[0];
// }else{
//// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeInputBug4";
// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/allDataset";
//// outputPath = "/Users/anilkoyuncu/bugStudy/code/python/GumTreeOutput2/";
// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputSingle";
// }
// List<String> pjList = Arrays.asList("DATAJPA","ZXing","PDE","SWS","SWT", "SWF", "COLLECTIONS", "JDT");
List<String> files = new ArrayList<String>();
// files.add("5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.java");
files.add("d6c5e5_9f96d9_hbase-server#src#main#java#org#apache#hadoop#hbase#master#RegionStates.java");
for(String f : files){
String pjName = "HBASE";
// for (File target : folders) {
// String pjName = target.getName();
// if (!pjList.contains(pjName)){
// continue;
// }
// final List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
MessageFile messageFile = getMessageFile(inputPath + "/" + pjName +"/", f);
List<MessageFile> msgFiles = new ArrayList<>();
msgFiles.add(messageFile);
String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/";
final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list";
final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list";
final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list";
final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv";
final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list";
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ActionSetDumps");
FileHelper.deleteDirectory(editScriptsFilePath);
FileHelper.deleteDirectory(patchesSourceCodeFilePath);
FileHelper.deleteDirectory(buggyTokensFilePath);
FileHelper.deleteDirectory(editScriptSizesFilePath);
FileHelper.deleteDirectory(alarmTypesFilePath);
StringBuilder astEditScripts = new StringBuilder();
StringBuilder tokens = new StringBuilder();
StringBuilder sizes = new StringBuilder();
StringBuilder patches = new StringBuilder();
StringBuilder alarmTypes = new StringBuilder();
// int a = 0;
ActorSystem system = null;
ActorRef parsingActor = null;
final EDiffMessage msg = new EDiffMessage(0, msgFiles);
try {
log.info("Akka begins...");
system = ActorSystem.create("Mining-FixPattern-System");
parsingActor = system.actorOf(EDiffActor.props(1, "dataset"), "mine-fix-pattern-actor");
parsingActor.tell(msg, ActorRef.noSender());
} catch (Exception e) {
system.shutdown();
e.printStackTrace();
}
// int counter = 0;
// for (MessageFile msgFile : msgFiles) {
// FixedPatternHunkParser parser = new FixedPatternHunkParser();
//
// final ExecutorService executor = Executors.newSingleThreadExecutor();
// // schedule the work
// final Future<?> future = executor.submit(new RunnableParser(msgFile.getPrevFile(),
// msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser));
// try {
// // where we wait for task to complete
// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS);
// String editScripts = parser.getAstEditScripts();
// if (!editScripts.equals("")) {
// astEditScripts.append(editScripts);
// tokens.append(parser.getTokensOfSourceCode());
// sizes.append(parser.getSizes());
// patches.append(parser.getPatchesSourceCode());
// alarmTypes.append(parser.getAlarmTypes());
// //5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.txt_1_CAMEL
//
// a++;
// if (a % 100 == 0) {
// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true);
// FileHelper.outputToFile(buggyTokensFilePath, tokens, true);
// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true);
// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true);
// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true);
// astEditScripts.setLength(0);
// tokens.setLength(0);
// sizes.setLength(0);
// patches.setLength(0);
// alarmTypes.setLength(0);
// System.out.println("Finish of parsing " + a + " files......");
// }
// }
// } catch (TimeoutException e) {
// err.println("task timed out");
// future.cancel(true /* mayInterruptIfRunning */);
// } catch (InterruptedException e) {
// err.println("task interrupted");
// } catch (ExecutionException e) {
// err.println("task aborted");
// } finally {
// executor.shutdownNow();
// }
// }
FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true);
FileHelper.outputToFile(buggyTokensFilePath, tokens, true);
FileHelper.outputToFile(editScriptSizesFilePath, sizes, true);
FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true);
FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true);
astEditScripts.setLength(0);
tokens.setLength(0);
sizes.setLength(0);
patches.setLength(0);
alarmTypes.setLength(0);
// classifyByAlarmTypes();
}
}
// File folder = new File(inputPath);
// File[] listOfFiles = folder.listFiles();
// Stream<File> stream = Arrays.stream(listOfFiles);
// List<File> folders = stream
// .filter(x -> !x.getName().startsWith("."))
//
// .collect(Collectors.toList());
//
//// List<File> targetList = new ArrayList<File>();
//// for (File f:folders){
//// for(File f1 :f.listFiles()){
//// if (!f1.getName().startsWith(".")){
//// targetList.add(f1);
//// }
//// }
//// }
//
//// List<String> pjList = Arrays.asList("DATAJPA","ZXing","PDE","SWS","SWT", "SWF", "COLLECTIONS", "JDT");
// List<String> files = new ArrayList<String>();
//// files.add("5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.java");
// files.add("d6c5e5_9f96d9_hbase-server#src#main#java#org#apache#hadoop#hbase#master#RegionStates.java");
// for(String f : files){
// String pjName = "HBASE";
//// for (File target : folders) {
//// String pjName = target.getName();
//// if (!pjList.contains(pjName)){
//// continue;
//// }
//
//// final List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
// MessageFile messageFile = getMessageFile(inputPath + "/" + pjName +"/", f);
//
// List<MessageFile> msgFiles = new ArrayList<>();
// msgFiles.add(messageFile);
// String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/";
// final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list";
// final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list";
// final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list";
// final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv";
// final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list";
//
//
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ActionSetDumps");
// FileHelper.deleteDirectory(editScriptsFilePath);
// FileHelper.deleteDirectory(patchesSourceCodeFilePath);
// FileHelper.deleteDirectory(buggyTokensFilePath);
// FileHelper.deleteDirectory(editScriptSizesFilePath);
// FileHelper.deleteDirectory(alarmTypesFilePath);
//
// StringBuilder astEditScripts = new StringBuilder();
// StringBuilder tokens = new StringBuilder();
// StringBuilder sizes = new StringBuilder();
// StringBuilder patches = new StringBuilder();
// StringBuilder alarmTypes = new StringBuilder();
//
//// int a = 0;
//
// ActorSystem system = null;
// ActorRef parsingActor = null;
// final EDiffMessage msg = new EDiffMessage(0, msgFiles);
// try {
// log.info("Akka begins...");
// system = ActorSystem.create("Mining-FixPattern-System");
// parsingActor = system.actorOf(EDiffActor.props(1, "dataset"), "mine-fix-pattern-actor");
// parsingActor.tell(msg, ActorRef.noSender());
// } catch (Exception e) {
// system.shutdown();
// e.printStackTrace();
// }
//
//// int counter = 0;
//// for (MessageFile msgFile : msgFiles) {
//// EDiffHunkParser parser = new EDiffHunkParser();
////
//// final ExecutorService executor = Executors.newSingleThreadExecutor();
//// // schedule the work
//// final Future<?> future = executor.submit(new RunnableParser(msgFile.getPrevFile(),
//// msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser));
//// try {
//// // where we wait for task to complete
//// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS);
//// String editScripts = parser.getAstEditScripts();
//// if (!editScripts.equals("")) {
//// astEditScripts.append(editScripts);
//// tokens.append(parser.getTokensOfSourceCode());
//// sizes.append(parser.getSizes());
//// patches.append(parser.getPatchesSourceCode());
//// alarmTypes.append(parser.getAlarmTypes());
////
//// a++;
//// if (a % 100 == 0) {
//// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true);
//// FileHelper.outputToFile(buggyTokensFilePath, tokens, true);
//// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true);
//// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true);
//// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true);
//// astEditScripts.setLength(0);
//// tokens.setLength(0);
//// sizes.setLength(0);
//// patches.setLength(0);
//// alarmTypes.setLength(0);
//// System.out.println("Finish of parsing " + a + " files......");
//// }
//// }
//// } catch (TimeoutException e) {
//// err.println("task timed out");
//// future.cancel(true /* mayInterruptIfRunning */);
//// } catch (InterruptedException e) {
//// err.println("task interrupted");
//// } catch (ExecutionException e) {
//// err.println("task aborted");
//// } finally {
//// executor.shutdownNow();
//// }
//// }
//
// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true);
// FileHelper.outputToFile(buggyTokensFilePath, tokens, true);
// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true);
// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true);
// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true);
// astEditScripts.setLength(0);
// tokens.setLength(0);
// sizes.setLength(0);
// patches.setLength(0);
// alarmTypes.setLength(0);
//
//
//// classifyByAlarmTypes();
// }
// }
private static List<MessageFile> getMessageFiles(String gumTreeInput) {
src/main/java/edu/lu/uni/serval/fixminer/akka/BaseMessage.java
+46
View File
@@ -0,0 +1,46 @@
package edu.lu.uni.serval.fixminer.akka;
/**
* Created by anilkoyuncu on 18/09/2018.
*/
public class BaseMessage {
private long SECONDS_TO_WAIT;
private int id;
private int threadPoolSize;
public BaseMessage(int id,Long timeout) {
this.id = id;
this.SECONDS_TO_WAIT = timeout;
// this.threadPoolSize = threadPoolSize;
}
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public long getSECONDS_TO_WAIT() {
return SECONDS_TO_WAIT;
}
public void setSECONDS_TO_WAIT(long SECONDS_TO_WAIT) {
this.SECONDS_TO_WAIT = SECONDS_TO_WAIT;
}
public int getThreadPoolSize() {
return threadPoolSize;
}
public void setThreadPoolSize(int threadPoolSize) {
this.threadPoolSize = threadPoolSize;
}
}
src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/AkkaTreeParser.java → src/main/java/edu/lu/uni/serval/fixminer/akka/compare/AkkaTreeParser.java
+5 -5
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.cluster.akka;
package edu.lu.uni.serval.fixminer.akka.compare;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
@@ -19,7 +19,7 @@ public class AkkaTreeParser {
private static Logger log = LoggerFactory.getLogger(AkkaTreeParser.class);
public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, String cursor){
public static void akkaCompare(JedisPool innerPool, JedisPool outerPool, String numOfWorkers, int cursor, String eDiffTimeout){
final List<String> listOfPairs = getMessages(innerPool,cursor); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
@@ -28,7 +28,7 @@ public class AkkaTreeParser {
ActorSystem system = null;
ActorRef parsingActor = null;
final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool);
final TreeMessage msg = new TreeMessage(0,listOfPairs, innerPool,outerPool,eDiffTimeout);
try {
log.info("Akka begins...");
system = ActorSystem.create("Compare-EnhancedDiff-System");
@@ -41,7 +41,7 @@ public class AkkaTreeParser {
}
}
public static List<String> getMessages(JedisPool innerPool, String cursor){
public static List<String> getMessages(JedisPool innerPool, int cursor){
ScanResult<String> scan;
@@ -54,7 +54,7 @@ public class AkkaTreeParser {
ScanParams sc = new ScanParams();
//150000000
log.info("Scanning ");
sc.count(Integer.valueOf(cursor));
sc.count(cursor);
sc.match("pair_[0-9]*");
src/main/java/edu/lu/uni/serval/fixminer/cluster/Compare.java → src/main/java/edu/lu/uni/serval/fixminer/akka/compare/Compare.java
+3 -3
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.cluster;
package edu.lu.uni.serval.fixminer.akka.compare;
import com.github.gumtreediff.actions.ActionGenerator;
import com.github.gumtreediff.actions.model.Action;
@@ -71,7 +71,7 @@ public class Compare {
if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
|| ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) {
String matchKey = "match_" + (String.valueOf(i)) + "_" + String.valueOf(j);
// log.info(matchKey);
log.info("{} tagged to be similar" ,matchKey);
jedis.select(1);
jedis.set(matchKey, result);
@@ -86,7 +86,7 @@ public class Compare {
} catch (Exception e) {
log.error(e.toString() + " {}", name);
log.debug("{} not comparable", name);
// e.printStackTrace();
src/main/java/edu/lu/uni/serval/fixminer/cluster/RunnableCompare.java → src/main/java/edu/lu/uni/serval/fixminer/akka/compare/RunnableCompare.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.cluster;
package edu.lu.uni.serval.fixminer.akka.compare;
import redis.clients.jedis.JedisPool;
src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeActor.java → src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeActor.java
+2 -2
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.cluster.akka;
package edu.lu.uni.serval.fixminer.akka.compare;
import akka.actor.ActorRef;
import akka.actor.Props;
@@ -60,7 +60,7 @@ public class TreeActor extends UntypedActor {
int toIndex = (i + 1) * average + counter;
List<String> pairsOfWorkers = pairs.subList(fromIndex, toIndex);
final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool);
final TreeMessage workMsg = new TreeMessage(i + 1, pairsOfWorkers,innerPool,outerPool,((TreeMessage) message).getSECONDS_TO_WAIT());
mineRouter.tell(workMsg, getSelf());
logger.info("Assign a task to worker #" + (i + 1) + "...");
}
src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeMessage.java → src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeMessage.java
+13 -10
View File
@@ -1,5 +1,6 @@
package edu.lu.uni.serval.fixminer.cluster.akka;
package edu.lu.uni.serval.fixminer.akka.compare;
import edu.lu.uni.serval.fixminer.akka.BaseMessage;
import redis.clients.jedis.JedisPool;
import java.util.List;
@@ -7,25 +8,27 @@ import java.util.List;
/**
* Created by anilkoyuncu on 12/09/2018.
*/
public class TreeMessage {
private int id;
public class TreeMessage extends BaseMessage{
private List<String> name;
private JedisPool innerPool;
private JedisPool outerPool;
public TreeMessage(int id, List<String> name, JedisPool innerPool, JedisPool outerPool) {
this.id = id;
public TreeMessage(int id, List<String> name, JedisPool innerPool, JedisPool outerPool,String eDiffTimeout) {
super(id,new Long(eDiffTimeout));
this.name = name;
this.innerPool = innerPool;
this.outerPool = outerPool;
}
public int getId() {
return id;
}
public TreeMessage(int id, List<String> name, JedisPool innerPool, JedisPool outerPool,Long eDiffTimeout) {
super(id,eDiffTimeout);
this.name = name;
this.innerPool = innerPool;
this.outerPool = outerPool;
public void setId(int id) {
this.id = id;
}
public List<String> getName() {
src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/TreeWorker.java → src/main/java/edu/lu/uni/serval/fixminer/akka/compare/TreeWorker.java
+6 -9
View File
@@ -1,11 +1,9 @@
package edu.lu.uni.serval.fixminer.cluster.akka;
package edu.lu.uni.serval.fixminer.akka.compare;
import akka.actor.Props;
import akka.actor.UntypedActor;
import akka.japi.Creator;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.fixminer.cluster.Compare;
import edu.lu.uni.serval.fixminer.cluster.RunnableCompare;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.JedisPool;
@@ -66,7 +64,7 @@ public class TreeWorker extends UntypedActor {
if(message instanceof TreeMessage) {
// if (message instanceof edu.lu.uni.serval.fixminer.cluster.akka.EDiffMessage) {
// if (message instanceof edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage) {
TreeMessage msg = (TreeMessage) message;
List<String> files = msg.getName();
JedisPool innerPool = msg.getInnerPool();
@@ -92,12 +90,12 @@ public class TreeWorker extends UntypedActor {
// }.counter;
//
final ExecutorService executor = Executors.newFixedThreadPool(20);
final ExecutorService executor = Executors.newWorkStealingPool();
// // schedule the work
final Future<?> future = executor.submit(new RunnableCompare(name, innerPool, outerPool, new Compare()));
try {
// wait for task to complete
future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS);
future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS);
Compare compare = new Compare();
compare.coreCompare(name, innerPool, outerPool);
counter++;
@@ -157,9 +155,8 @@ public class TreeWorker extends UntypedActor {
}
}
log.info("bitti");
log.info("Worker #" + id +"finialized parsing " + counter + " files...");
log.info("Worker #" + id + " finialized the work...");
// log.info("Worker #" + id +"finialized parsing " + counter + " files...");
log.info("Worker #" + id + " finalized the work...");
this.getSender().tell("STOP", getSelf());
// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
// stopServer = String.format(stopServer,Integer.valueOf(innerPort));
src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffActor.java → src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffActor.java
+4 -4
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.cluster.akka;
package edu.lu.uni.serval.fixminer.akka.ediff;
import akka.actor.ActorRef;
import akka.actor.Props;
@@ -54,15 +54,15 @@ public class EDiffActor extends UntypedActor {
int toIndex = (i + 1) * average + counter;
List<MessageFile> filesOfWorkers = files.subList(fromIndex, toIndex);
final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers);
final EDiffMessage workMsg = new EDiffMessage(i + 1, filesOfWorkers,((EDiffMessage) message).getSECONDS_TO_WAIT(),((EDiffMessage) message).getActionType());
mineRouter.tell(workMsg, getSelf());
logger.info("Assign a task to worker #" + (i + 1) + "...");
}
} else if ("STOP".equals(message.toString())) {
counter ++;
logger.info(counter + " workers finailized their work...");
logger.info(counter + " workers finalized their work...");
if (counter >= numberOfWorkers) {
logger.info("All workers finailized their work...");
logger.info("All workers finalized their work...");
this.getContext().stop(mineRouter);
this.getContext().stop(getSelf());
this.getContext().system().shutdown();
src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternHunkParser.java → src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java
+47 -23
View File
@@ -1,4 +1,9 @@
package edu.lu.uni.serval.FixPatternParser;
package edu.lu.uni.serval.fixminer.akka.ediff;
import com.github.gumtreediff.actions.model.Delete;
import com.github.gumtreediff.actions.model.Insert;
import com.github.gumtreediff.actions.model.Move;
import com.github.gumtreediff.actions.model.Update;
import java.io.*;
import java.util.List;
@@ -11,7 +16,7 @@ import java.util.List;
* @author kui.liu
*
*/
public class FixedPatternHunkParser extends FixedPatternParser {
public class EDiffHunkParser extends EDiffParser {
public String testingInfo = "";
@@ -25,30 +30,49 @@ public class FixedPatternHunkParser extends FixedPatternParser {
public String unfixedViolations = "";
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project) {
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project,String actionType) {
List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile);
if (actionSets.size() != 0) {
// boolean isUpdate =
// actionSets.stream().allMatch(p -> p.getAction() instanceof Update);
// boolean isInsert =
// actionSets.stream().allMatch(p -> p.getAction() instanceof Insert);
// boolean isDelete =
// actionSets.stream().allMatch(p -> p.getAction() instanceof Delete);
// boolean isMove =
// actionSets.stream().allMatch(p -> p.getAction() instanceof Move);
String folder= null;
boolean processActionSet = false;
switch (actionType){
case "ALL":
folder = "/ALL/";
processActionSet = true;
break;
case "UPD":
processActionSet =
actionSets.stream().allMatch(p -> p.getAction() instanceof Update);
folder = "/UPD/";
break;
case "INS":
processActionSet =
actionSets.stream().allMatch(p -> p.getAction() instanceof Insert);
folder = "/INS/";
break;
case "DEL":
processActionSet =
actionSets.stream().allMatch(p -> p.getAction() instanceof Delete);
folder = "/DEL/";
break;
case "MOV":
processActionSet =
actionSets.stream().allMatch(p -> p.getAction() instanceof Move);
folder = "/MOV/";
break;
default:
processActionSet = false;
System.err.print(actionType + "not known");
break;
}
int hunkSet = 0;
// if (isUpdate || isInsert || isDelete || isMove) {
if(processActionSet){
for (HierarchicalActionSet actionSet : actionSets) {
String folder = "/ALL/";
// if (isUpdate) {
// folder = "/UPD/";
// } else if (isDelete) {
// folder = "/DEL/";
// } else if (isInsert) {
// folder = "/INS/";
// } else if (isMove) {
// folder = "/MOV/";
// }
FileOutputStream f = null;
@@ -75,7 +99,7 @@ public class FixedPatternHunkParser extends FixedPatternParser {
hunkSet++;
}
// }
}
}
}
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffMessage.java
+45
View File
@@ -0,0 +1,45 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
import edu.lu.uni.serval.fixminer.akka.BaseMessage;
import java.util.List;
public class EDiffMessage extends BaseMessage{
private List<MessageFile> msgFiles;
private String actionType;
public EDiffMessage(int id, List<MessageFile> msgFiles,String eDiffTimeout,String actionType) {
super(id,new Long(eDiffTimeout));
this.msgFiles = msgFiles;
this.actionType = actionType;
}
public EDiffMessage(int id, List<MessageFile> msgFiles,Long eDiffTimeout,String actionType) {
super(id,eDiffTimeout);
this.msgFiles = msgFiles;
this.actionType = actionType;
}
public String getActionType() {
return actionType;
}
public void setActionType(String actionType) {
this.actionType = actionType;
}
public List<MessageFile> getMsgFiles() {
return msgFiles;
}
}
src/main/java/edu/lu/uni/serval/FixPatternParser/FixedPatternParser.java → src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java
+6 -15
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.FixPatternParser;
package edu.lu.uni.serval.fixminer.akka.ediff;
import com.github.gumtreediff.actions.model.Action;
import edu.lu.uni.serval.gumtree.GumTreeComparer;
@@ -14,7 +14,7 @@ import java.util.List;
* @author kui.liu
*
*/
public class FixedPatternParser extends Parser {
public class EDiffParser extends Parser {
/*
* ResultType:
@@ -26,11 +26,7 @@ public class FixedPatternParser extends Parser {
*/
public int resultType = 0;
protected String violationTypes = "";
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffentryFile,String project) {
}
/**
* Regroup GumTree results without remove the modification of variable names.
@@ -65,16 +61,11 @@ public class FixedPatternParser extends Parser {
}
}
@Override
public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile, String project, String actionType) {
}
// @Override
// public void parseFixPatterns(File prevFile, File revFile, File diffEntryFile) {
//
// }
// public void setUselessViolations(List<Violation> uselessViolations) {
// this.uselessViolations = uselessViolations;
// }
}
src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffWorker.java → src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffWorker.java
+9 -11
View File
@@ -1,11 +1,8 @@
package edu.lu.uni.serval.fixminer.cluster.akka;
package edu.lu.uni.serval.fixminer.akka.ediff;
import akka.actor.Props;
import akka.actor.UntypedActor;
import akka.japi.Creator;
import edu.lu.uni.serval.FixPatternParser.FixedPatternHunkParser;
import edu.lu.uni.serval.FixPatternParser.RunnableParser;
import edu.lu.uni.serval.config.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,14 +65,15 @@ public class EDiffWorker extends UntypedActor {
File diffentryFile = msgFile.getDiffEntryFile();
FixedPatternHunkParser parser = new FixedPatternHunkParser();
EDiffHunkParser parser = new EDiffHunkParser();
final ExecutorService executor = Executors.newSingleThreadExecutor();
final ExecutorService executor = Executors.newWorkStealingPool();
// schedule the work
final Future<?> future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project));
final Future<?> future = executor.submit(new RunnableParser(prevFile, revFile, diffentryFile, parser,project,msg.getActionType()));
try {
// wait for task to complete
future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS);
future.get(msg.getSECONDS_TO_WAIT(), TimeUnit.SECONDS);
nullDiffEntry += parser.nullMatchedDiffEntry;
nullMappingGumTreeResults += parser.nullMappingGumTreeResult;
@@ -101,7 +99,7 @@ public class EDiffWorker extends UntypedActor {
patchesSourceCode.setLength(0);
sizes.setLength(0);
tokens.setLength(0);
log.info("Worker #" + id +" finialized parsing " + counter + " files...");
log.info("Worker #" + id +" finalized parsing " + counter + " files...");
testingInfo.setLength(0);
}
}
@@ -128,8 +126,8 @@ public class EDiffWorker extends UntypedActor {
testingInfo.setLength(0);
}
log.info("Worker #" + id +"finialized parsing " + counter + " files...");
log.info("Worker #" + id + " finialized the work...");
// log.info("Worker #" + id +" finalized parsing " + counter + " files...");
log.info("Worker #" + id + " finalized the work...");
this.getSender().tell("STOP", getSelf());
} else {
unhandled(message);
src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalActionSet.java → src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalActionSet.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.FixPatternParser;
package edu.lu.uni.serval.fixminer.akka.ediff;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.tree.ITree;
src/main/java/edu/lu/uni/serval/FixPatternParser/HierarchicalRegrouper.java → src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouper.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.FixPatternParser;
package edu.lu.uni.serval.fixminer.akka.ediff;
import com.github.gumtreediff.actions.model.*;
import com.github.gumtreediff.tree.ITree;
src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/MessageFile.java → src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/MessageFile.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.cluster.akka;
package edu.lu.uni.serval.fixminer.akka.ediff;
import java.io.File;
src/main/java/edu/lu/uni/serval/FixPatternParser/Parser.java → src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/Parser.java
+2 -2
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.FixPatternParser;
package edu.lu.uni.serval.fixminer.akka.ediff;
import java.io.File;
@@ -18,7 +18,7 @@ public abstract class Parser implements ParserInterface {
protected String originalTree = ""; // Guide of generating patches.
protected String actionSets = ""; // Guide of generating patches.
public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile,String project);
public abstract void parseFixPatterns(File prevFile, File revFile, File diffEntryFile,String project,String actionType);
@Override
src/main/java/edu/lu/uni/serval/FixPatternParser/ParserInterface.java → src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/ParserInterface.java
+1 -1
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.FixPatternParser;
package edu.lu.uni.serval.fixminer.akka.ediff;
public interface ParserInterface {
src/main/java/edu/lu/uni/serval/FixPatternParser/RunnableParser.java → src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/RunnableParser.java
+5 -3
View File
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.FixPatternParser;
package edu.lu.uni.serval.fixminer.akka.ediff;
import java.io.File;
@@ -9,6 +9,7 @@ public class RunnableParser implements Runnable {
private File diffentryFile;
private Parser parser;
private String project;
private String actionType;
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser) {
this.prevFile = prevFile;
@@ -17,16 +18,17 @@ public class RunnableParser implements Runnable {
this.parser = parser;
}
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser,String project) {
public RunnableParser(File prevFile, File revFile, File diffentryFile, Parser parser,String project,String actionType) {
this.prevFile = prevFile;
this.revFile = revFile;
this.diffentryFile = diffentryFile;
this.parser = parser;
this.project = project;
this.actionType= actionType;
}
@Override
public void run() {
parser.parseFixPatterns(prevFile, revFile, diffentryFile,project);
parser.parseFixPatterns(prevFile, revFile, diffentryFile,project,actionType);
}
}
src/main/java/edu/lu/uni/serval/fixminer/cluster/akka/EDiffMessage.java
-24
View File
@@ -1,24 +0,0 @@
package edu.lu.uni.serval.fixminer.cluster.akka;
import java.util.List;
public class EDiffMessage {
private int id;
private List<MessageFile> msgFiles;
public EDiffMessage(int id, List<MessageFile> msgFiles) {
super();
this.id = id;
this.msgFiles = msgFiles;
}
public int getId() {
return id;
}
public List<MessageFile> getMsgFiles() {
return msgFiles;
}
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/AkkaTreeLoader.java
+13 -3
View File
@@ -14,7 +14,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
import static edu.lu.uni.serval.fixminer.cluster.akka.AkkaTreeParser.akkaCompare;
import static edu.lu.uni.serval.fixminer.akka.compare.AkkaTreeParser.akkaCompare;
/**
* Created by anilkoyuncu on 19/03/2018.
@@ -24,7 +24,7 @@ public class AkkaTreeLoader {
private static Logger log = LoggerFactory.getLogger(AkkaTreeLoader.class);
public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, String cursor,String chunk) throws Exception {
public static void main(String portInner, String dbDir, String chunkName, String port, String dumpsName, String pairsPath, String numOfWorkers, int cursor,String chunk,String eDiffTimeout) throws Exception {
String parameters = String.format("\nportInner %s \nchunkName %s \ndbDir %s \ndumpsName %s",portInner,chunkName,dbDir,dumpsName);
@@ -90,7 +90,17 @@ public class AkkaTreeLoader {
akkaCompare(innerPool,outerPool,numOfWorkers,cursor);
akkaCompare(innerPool,outerPool,numOfWorkers,cursor,eDiffTimeout);
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
String stopServer2 = String.format(stopServer,Integer.valueOf(port));
cs.runShell(stopServer2,port);
String stopServer1 = String.format(stopServer,Integer.valueOf(portInner));
cs.runShell(stopServer1,portInner);
src/main/java/edu/lu/uni/serval/fixminer/jobs/CalculatePairs.java
+53 -33
View File
@@ -1,8 +1,8 @@
package edu.lu.uni.serval.fixminer.jobs;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.PoolBuilder;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
@@ -22,7 +22,7 @@ import java.util.List;
public class CalculatePairs {
private static Logger log = LoggerFactory.getLogger(CalculatePairs.class);
public static void main(String dbDir,String chunkName,String port,String outputPath,String pjName) throws Exception {
public static void main(String dbDir,String chunkName,String port,String outputPath,String pjName,boolean isBigPair,int cursor) throws Exception {
String parameters = String.format("\nport %s \nchunkName %s \ndbDir %s",port,chunkName,dbDir);
@@ -46,7 +46,7 @@ public class CalculatePairs {
ScanParams sc = new ScanParams();
//150000000
sc.count(1500000000);
sc.count(cursor);
sc.match("*");
scan = outer.scan("0", sc);
@@ -60,31 +60,59 @@ public class CalculatePairs {
byte [] buf = new byte[0];
String line = null;
if(isBigPair) {
bigPair(outputPath, pjName, result, buf, line);
}else{
smallPair(outputPath, pjName, result, buf, line);
}
// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
// String stopServer2 = String.format(stopServer,Integer.valueOf(port));
//// loadRedis(stopServer2,serverWait);
// cs.runShell(stopServer2,serverWait);
log.info("Done pairs");
}
private static void smallPair(String outputPath, String pjName, List<String> result, byte[] buf, String line) {
try {
// FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv");
// DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos));
//
FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv");
DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos));
FileOutputStream fosIndex = new FileOutputStream(outputPath + "/" +pjName+".index");
DataOutputStream outStreamIndex = new DataOutputStream(new BufferedOutputStream(fosIndex));
for (int i = 0; i < result.size(); i++) {
line = String.valueOf(i) +"," + result.get(i)+"\n";
outStreamIndex.write(line.getBytes());
for (int j = i + 1; j < result.size(); j++) {
line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n";
outStream.write(line.getBytes());
}
}
outStream.close();
outStreamIndex.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}catch (java.nio.BufferOverflowException e) {
log.error(line);
log.error(String.valueOf(buf.length));
e.printStackTrace();
}
}
private static void bigPair(String outputPath, String pjName, List<String> result, byte[] buf, String line) {
try {
FileOutputStream fosIndex = new FileOutputStream(outputPath + "/" +pjName+".index");
DataOutputStream outStreamIndex = new DataOutputStream(new BufferedOutputStream(fosIndex));
//
//
//
//
// for (int i = 0; i < result.size(); i++) {
// line = String.valueOf(i) +"," + result.get(i)+"\n";
// outStreamIndex.write(line.getBytes());
//
// for (int j = i + 1; j < result.size(); j++) {
//
//
//
// line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n";
// outStream.write(line.getBytes());
//
// }
// }
// outStream.close();
// outStreamIndex.close();
int fileCounter = 0;
FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +pjName +String.valueOf(fileCounter)+".txt", "rw").getChannel();
int maxSize = 500*1000000;
@@ -97,8 +125,6 @@ public class CalculatePairs {
for (int j = i + 1; j < result.size(); j++) {
// line = String.valueOf(i) +"\t" + String.valueOf(j)+"\n"; // + "\t" + result.get(i) + "\t" + result.get(j)+"\n";
line = String.valueOf(i) +"," + String.valueOf(j)+"\n"; // + "," + result.get(i) + "," + result.get(j)+"\n";
buf = line.getBytes();
if(wrBuf.remaining() > 500) {
@@ -126,12 +152,6 @@ public class CalculatePairs {
log.error(String.valueOf(buf.length));
e.printStackTrace();
}
// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
// String stopServer2 = String.format(stopServer,Integer.valueOf(port));
//// loadRedis(stopServer2,serverWait);
// cs.runShell(stopServer2,serverWait);
log.info("Done pairs");
}
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
+11 -10
View File
@@ -2,9 +2,9 @@ package edu.lu.uni.serval.fixminer.jobs;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import edu.lu.uni.serval.fixminer.cluster.akka.EDiffMessage;
import edu.lu.uni.serval.fixminer.cluster.akka.MessageFile;
import edu.lu.uni.serval.fixminer.cluster.akka.EDiffActor;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffActor;
import edu.lu.uni.serval.fixminer.akka.ediff.EDiffMessage;
import edu.lu.uni.serval.fixminer.akka.ediff.MessageFile;
import edu.lu.uni.serval.utils.FileHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -20,7 +20,7 @@ public class EnhancedASTDiff {
private static Logger log = LoggerFactory.getLogger(EnhancedASTDiff.class);
public static void main(String inputPath, String outputPath,String numOfWorkers,String project) {
public static void main(String inputPath, String outputPath, String numOfWorkers, String project, String eDiffTimeout, String actionType) {
String parameters = String.format("\nInput path %s \nOutput path %s",inputPath,outputPath);
@@ -45,18 +45,19 @@ public class EnhancedASTDiff {
String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/";
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD");
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS");
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL");
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV");
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ALL");
// a
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/UPD");
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS");
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL");
// FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV");
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/"+actionType);
int a = 0;
ActorSystem system = null;
ActorRef parsingActor = null;
final EDiffMessage msg = new EDiffMessage(0, msgFiles);
final EDiffMessage msg = new EDiffMessage(0, msgFiles,eDiffTimeout,actionType);
try {
log.info("Akka begins...");
system = ActorSystem.create("Mining-FixPattern-System");
src/main/java/edu/lu/uni/serval/fixminer/jobs/ImportPairs2DB.java
+1 -1
View File
@@ -44,7 +44,7 @@ public class ImportPairs2DB {
cmd = String.format(cmd, pj.getPath(), portInt);
log.info(cmd);
cs.runShell(cmd);
cs.runShell(cmd,portInner);
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
String stopServer2 = String.format(stopServer,portInt);
src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster.java
+9 -12
View File
@@ -6,12 +6,8 @@ import com.github.gumtreediff.matchers.Matcher;
import com.github.gumtreediff.matchers.Matchers;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiff;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.utils.*;
import org.javatuples.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,7 +32,7 @@ public class MultiThreadTreeLoaderCluster {
private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoaderCluster.class);
public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type) throws Exception {
public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type,int cursor) throws Exception {
CallShell cs = new CallShell();
String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
@@ -74,7 +70,7 @@ public class MultiThreadTreeLoaderCluster {
try (Jedis jedis = jedisPool.getResource()) {
// do operations with jedis resource
ScanParams sc = new ScanParams();
sc.count(150000000);
sc.count(cursor);
sc.match("pair_[0-9]*");
log.info("Scanning");
@@ -84,13 +80,14 @@ public class MultiThreadTreeLoaderCluster {
if (size == 0) {
String comd = String.format(cmd3,f.getPath(),portInner);
// loadRedis(comd);
cs.runShell(comd);
log.info("Importing {} pairs for cluster {}",size,f.getName());
cs.runShell(comd,portInner);
scan = jedis.scan("0", sc);
size = scan.getResult().size();
}
log.info("Scanned " + String.valueOf(size));
log.info("Scanned {} for cluster {}",String.valueOf(size),f.getName());
String clusterName = f.getName().replaceAll("[^0-9]+", "");
@@ -247,7 +244,7 @@ public class MultiThreadTreeLoaderCluster {
// log.info("Completed " + resultKey);
}catch (Exception e){
log.warn(e.toString() + " {}",(name));
log.debug(e.toString() + " {}",(name));
}
@@ -329,7 +326,7 @@ public class MultiThreadTreeLoaderCluster {
e.printStackTrace();
}
log.info("Done pairs");
log.info("Done pairs of cluster {}",cluster);
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/MultiThreadTreeLoaderCluster3.java
+10 -9
View File
@@ -2,11 +2,11 @@ package edu.lu.uni.serval.fixminer.jobs;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.similarity.JaroWinklerDistance;
import org.slf4j.Logger;
@@ -33,7 +33,7 @@ public class MultiThreadTreeLoaderCluster3 {
private static Logger log = LoggerFactory.getLogger(MultiThreadTreeLoaderCluster3.class);
public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type) throws Exception {
public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String type,int cursor) throws Exception {
CallShell cs = new CallShell();
String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
@@ -69,7 +69,7 @@ public class MultiThreadTreeLoaderCluster3 {
try (Jedis jedis = jedisPool.getResource()) {
// do operations with jedis resource
ScanParams sc = new ScanParams();
sc.count(150000000);
sc.count(cursor);
sc.match("pair_[0-9]*");
log.info("Scanning");
@@ -78,7 +78,7 @@ public class MultiThreadTreeLoaderCluster3 {
if (size == 0) {
String comd = String.format(cmd3, f.getPath(), portInner);
cs.runShell(comd);
cs.runShell(comd,portInner);
scan = jedis.scan("0", sc);
@@ -149,7 +149,7 @@ public class MultiThreadTreeLoaderCluster3 {
ITree parent = null;
ITree children =null;
TreeContext tc = new TreeContext();
tree = EDiffHelper.getASTTree(actionSet, parent, children,tc);
tree = EDiffHelper.getTokenTree(actionSet, parent, children,tc);
// tree.setParent(null);
tc.validate();
@@ -526,7 +526,7 @@ public class MultiThreadTreeLoaderCluster3 {
.collect(Collectors.toList());
FileHelper.createDirectory(outputPath + "/pairs-2l"+type);
log.info("Calculating pairs");
for (File pj : pjs) {
File[] files = pj.listFiles();
List<File> fileList = Arrays.asList(files);
@@ -536,6 +536,7 @@ public class MultiThreadTreeLoaderCluster3 {
}
File[] clusterFiles = cluster.listFiles();
List<File> clusterFilesL = Arrays.asList(clusterFiles);
readMessageFilesCluster(clusterFilesL, outputPath, inputPath, pj.getName(), cluster.getName(),type);
@@ -559,7 +560,7 @@ public class MultiThreadTreeLoaderCluster3 {
treesFileNames.add(target.toString());
}
log.info("Calculating pairs");
// treesFileNames = treesFileNames.subList(0,100);
String filename = "cluster" + cluster + "_" + subCluster;
@@ -609,7 +610,7 @@ public class MultiThreadTreeLoaderCluster3 {
e.printStackTrace();
}
log.info("Done pairs");
log.info("Done pairs of {}",filename);
}
src/main/java/edu/lu/uni/serval/fixminer/jobs/PatternExtractor.java
+1 -1
View File
@@ -1,7 +1,7 @@
package edu.lu.uni.serval.fixminer.jobs;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.utils.Checker;
import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
src/main/java/edu/lu/uni/serval/fixminer/jobs/StoreEDiffInCache.java
+3 -3
View File
@@ -1,9 +1,9 @@
package edu.lu.uni.serval.fixminer.jobs;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
@@ -112,10 +112,10 @@ public class StoreEDiffInCache {
} catch (FileNotFoundException e) {
log.error("File not found");
log.error("File not found {}" , path);
e.printStackTrace();
} catch (IOException e) {
log.error("Error initializing stream");
log.error("Error initializing stream {}" , path);
e.printStackTrace();
} catch (ClassNotFoundException e) {
// TODO Auto-generated catch block
src/main/java/edu/lu/uni/serval/fixminer/jobs/TreeLoaderClusterL1.java
+2 -2
View File
@@ -22,7 +22,7 @@ public class TreeLoaderClusterL1 {
private static Logger log = LoggerFactory.getLogger(TreeLoaderClusterL1.class);
public static void main(String portInner,String port,String inputPath,String level1DB,String level1Path,String innerTypePrefix) throws Exception {
public static void main(String portInner,String port,String inputPath,String level1DB,String level1Path,String innerTypePrefix,int cursor) throws Exception {
@@ -59,7 +59,7 @@ public class TreeLoaderClusterL1 {
inner.select(1);
ScanParams sc = new ScanParams();
//150000000
sc.count(150000000);
sc.count(cursor);
sc.match("match_[0-9]*");
ScanResult<String> scan; scan = inner.scan("0", sc);
src/main/java/edu/lu/uni/serval/utils/CallShell.java
+6
View File
@@ -4,6 +4,9 @@ package edu.lu.uni.serval.utils;
* Created by anilkoyuncu on 17/04/2018.
*/
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -11,6 +14,7 @@ import java.util.concurrent.TimeUnit;
public class CallShell {
private static Logger log = LoggerFactory.getLogger(CallShell.class);
@@ -28,6 +32,8 @@ public class CallShell {
public static void runShell(String command, String port) throws Exception {
log.info(command);
Process process = Runtime.getRuntime().exec(command);
BufferedReader reader = new BufferedReader(new InputStreamReader(
process.getInputStream()));
src/main/java/edu/lu/uni/serval/utils/EDiff.java
+1 -1
View File
@@ -3,7 +3,7 @@ package edu.lu.uni.serval.utils;
import com.github.gumtreediff.actions.model.*;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
src/main/java/edu/lu/uni/serval/utils/EDiffHelper.java
+42 -1
View File
@@ -2,7 +2,7 @@ package edu.lu.uni.serval.utils;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import edu.lu.uni.serval.FixPatternParser.HierarchicalActionSet;
import edu.lu.uni.serval.fixminer.akka.ediff.HierarchicalActionSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,6 +38,47 @@ public class EDiffHelper {
return Base64.getEncoder().encodeToString(baos.toByteArray());
}
public static ITree getTokenTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){
int newType = 0;
String astNodeType = actionSet.getAstNodeType();
String label = actionSet.getAction().toString();
List<Integer> keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType);
if(keysByValue.size() != 1){
log.error("More than 1");
}
newType = keysByValue.get(0);
if(actionSet.getParent() == null){
//root
parent = tc.createTree(newType, label, null);
tc.setRoot(parent);
}else{
children = tc.createTree(newType, label, null);
children.setParentAndUpdateChildren(parent);
}
List<HierarchicalActionSet> subActions = actionSet.getSubActions();
if (subActions.size() != 0){
for (HierarchicalActionSet subAction : subActions) {
if(actionSet.getParent() == null){
children = parent;
}
getTokenTree(subAction,children,null,tc);
}
}
return parent;
}
public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children, TreeContext tc){
int newType = 0;