azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changes

Browse Source
This commit is contained in:
ANIL KOYUNCU
2020-01-21 16:17:47 +01:00
parent e3d2e59e4f
commit 5254e8e98d
4 changed files with 68 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+11 -10
View File
@@ -22,8 +22,8 @@ public class Launcher {
Properties appProps = new Properties(); Properties appProps = new Properties();
String appConfigPath = "/Users/anil.koyuncu/projects/fixminer/fixminer_source/src/main/resource/app.properties"; // String appConfigPath = "/Users/anil.koyuncu/projects/fixminer/fixminer_source/src/main/resource/app.properties";
// String appConfigPath = args[0]; String appConfigPath = args[0];
appProps.load(new FileInputStream(appConfigPath)); appProps.load(new FileInputStream(appConfigPath));
// String portInner = appProps.getProperty("portInner","6380"); // String portInner = appProps.getProperty("portInner","6380");
@@ -33,17 +33,18 @@ public class Launcher {
String actionType = appProps.getProperty("actionType","ALL"); String actionType = appProps.getProperty("actionType","ALL");
String eDiffTimeout = appProps.getProperty("eDiffTimeout","900"); String eDiffTimeout = appProps.getProperty("eDiffTimeout","900");
String parallelism = appProps.getProperty("parallelism","FORKJOIN"); String parallelism = appProps.getProperty("parallelism","FORKJOIN");
String hostname = appProps.getProperty("hostname","localhost");
String input = appProps.getProperty("inputPath","FORKJOIN"); String input = appProps.getProperty("inputPath","FORKJOIN");
String redisPath = appProps.getProperty("redisPath","FORKJOIN"); String redisPath = appProps.getProperty("redisPath","FORKJOIN");
String srcMLPath = appProps.getProperty("srcMLPath","FORKJOIN"); String srcMLPath = appProps.getProperty("srcMLPath","FORKJOIN");
// String parameter = args[2]; String parameter = args[2];
// String parameter = null; // String parameter = "L1";
// String parameter = "if"; // String parameter = "if";
String parameter = "add"; // String parameter = "add";
// String jobType = args[1]; String jobType = args[1];
String jobType = "RICHEDITSCRIPT"; // String jobType = "RICHEDITSCRIPT";
// String jobType = "LOAD"; // String jobType = "LOAD";
// String jobType = "COMPARE"; // String jobType = "COMPARE";
@@ -56,12 +57,12 @@ public class Launcher {
// //
// log.info(parameters); // log.info(parameters);
mainLaunch( numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter, srcMLPath); mainLaunch( numOfWorkers, jobType, portDumps, pjName,actionType,eDiffTimeout,parallelism,input,redisPath,parameter, srcMLPath,hostname);
} }
public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter,String srcMLPath){ public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String pjName, String actionType, String eDiffTimeout, String parallelism,String input, String redisPath,String parameter,String srcMLPath,String hostname){
String dbDir; String dbDir;
@@ -106,7 +107,7 @@ public class Launcher {
} }
CompareTrees.main(redisPath, portDumps,actionType+dumpsName, job); CompareTrees.main(redisPath, portDumps,actionType+dumpsName, job,numOfWorkers,hostname);
break; break;
case "PATTERN": case "PATTERN":
ClusterToPattern.main(portDumps,redisPath, actionType+dumpsName, parameter); ClusterToPattern.main(portDumps,redisPath, actionType+dumpsName, parameter);
src/main/java/edu/lu/uni/serval/fixminer/akka/compare/CompareTrees.java
+33 -19
View File
@@ -14,6 +14,7 @@ import redis.clients.jedis.JedisPool;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
@@ -29,7 +30,7 @@ public class CompareTrees {
private static Logger log = LoggerFactory.getLogger(CompareTrees.class); private static Logger log = LoggerFactory.getLogger(CompareTrees.class);
public static void main(String redisPath, String portDumps, String dumpsName, String job) throws Exception { public static void main(String redisPath, String portDumps, String dumpsName, String job,String numOfWorkers,String host) throws Exception {
// shape /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis ALLdumps-gumInput.rdb clusterl0-gumInputALL.rdb /Users/anil.koyuncu/projects/test/fixminer-core/python/data/richEditScript // shape /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis ALLdumps-gumInput.rdb clusterl0-gumInputALL.rdb /Users/anil.koyuncu/projects/test/fixminer-core/python/data/richEditScript
@@ -47,7 +48,7 @@ public class CompareTrees {
// cs.runShell(cmdInner, portInner); // cs.runShell(cmdInner, portInner);
// String numOfWorkers = "100000000";//args[4]; // String numOfWorkers = "100000000";//args[4];
String host = "localhost";//args[5]; // String host = "localhost";//args[5];
// -Djava.util.concurrent.ForkJoinPool.common.parallelism=256 // -Djava.util.concurrent.ForkJoinPool.common.parallelism=256
// final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000); // final JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), host,Integer.valueOf(portInner),20000000);
@@ -62,26 +63,36 @@ public class CompareTrees {
ArrayList<String> samePairs = new ArrayList<>(); ArrayList<String> samePairs = new ArrayList<>();
ArrayList<String> errorPairs = new ArrayList<>(); ArrayList<String> errorPairs = new ArrayList<>();
Integer numberOfWorkers = Integer.valueOf(numOfWorkers);
final ExecutorService executor = Executors.newWorkStealingPool(numberOfWorkers);
ArrayList<Future<?>> results = new ArrayList<Future<?>>();
for (int i = 1; i <numberOfWorkers ; i++) {
final ExecutorService executor = Executors.newWorkStealingPool();
// schedule the work
final Future<?> future = executor.submit( new RunnableCompare( job,errorPairs,filenames,outerPool)); // schedule the work
log.info("Starting job {}",i);
try { final Future<?> future = executor.submit(new RunnableCompare(job, errorPairs, filenames, outerPool, i));
// wait for task to complete results.add(future);
future.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
} finally {
executor.shutdownNow();
} }
for (Future<?> future:results){
try {
// wait for task to complete
future.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
} finally {
executor.shutdownNow();
}
}
log.info("End process"); log.info("End process");
@@ -93,12 +104,14 @@ public class CompareTrees {
ArrayList<String> errorPairs; ArrayList<String> errorPairs;
HashMap<String, String> filenames; HashMap<String, String> filenames;
JedisPool outerPool; JedisPool outerPool;
Integer threadID;
public RunnableCompare(String treeType,ArrayList<String> errorPairs, HashMap<String, String> filenames,JedisPool outerPool) { public RunnableCompare(String treeType,ArrayList<String> errorPairs, HashMap<String, String> filenames,JedisPool outerPool,Integer threadID) {
this.job = treeType; this.job = treeType;
this.errorPairs = errorPairs; this.errorPairs = errorPairs;
this.filenames = filenames; this.filenames = filenames;
this.outerPool = outerPool; this.outerPool = outerPool;
this.threadID = threadID;
} }
@Override @Override
@@ -113,6 +126,7 @@ public class CompareTrees {
stop = newCoreCompare(job, errorPairs, filenames, outerPool); stop = newCoreCompare(job, errorPairs, filenames, outerPool);
// } // }
} }
log.info("Completed worker {}",threadID);
} }
} }
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffHunkParser.java
+3 -3
View File
@@ -40,9 +40,9 @@ public class EDiffHunkParser extends EDiffParser {
try { try {
String astNodeType = actionSet.getAstNodeType(); String astNodeType = actionSet.getAstNodeType();
if (astNodeType.equals(rootType)){ // if (astNodeType.equals(rootType)){
//
} // }
actionSet.toString(); actionSet.toString();
int size = actionSet.getActionSize(); int size = actionSet.getActionSize();
src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
+21 -6
View File
@@ -34,16 +34,29 @@ public class EnhancedASTDiff {
CallShell cs = new CallShell(); CallShell cs = new CallShell();
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
if (rootType == null){ // if (rootType == null){
cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner)); cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
}else{ // }else{
cmd = String.format(cmd, dbDir,rootType+chunkName,Integer.valueOf(portInner)); // cmd = String.format(cmd, dbDir,rootType+chunkName,Integer.valueOf(portInner));
} // }
cs.runShell(cmd, portInner); cs.runShell(cmd, portInner);
JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000); JedisPool innerPool = new JedisPool(PoolBuilder.getPoolConfig(), "127.0.0.1",Integer.valueOf(portInner),20000000);
if (rootType == "add"){
try (Jedis inner = innerPool.getResource()) {
inner.select(2);
inner.flushDB();
inner.select(1);
inner.flushDB();
inner.select(0);
inner.del("compare");
}
}
File folder = new File(inputPath); File folder = new File(inputPath);
File[] listOfFiles = folder.listFiles(); File[] listOfFiles = folder.listFiles();
Stream<File> stream = Arrays.stream(listOfFiles); Stream<File> stream = Arrays.stream(listOfFiles);
@@ -79,8 +92,10 @@ public class EnhancedASTDiff {
parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor"); parsingActor = system.actorOf(EDiffActor.props(Integer.valueOf(numOfWorkers), project), "mine-fix-pattern-actor");
parsingActor.tell(msg, ActorRef.noSender()); parsingActor.tell(msg, ActorRef.noSender());
} catch (Exception e) { } catch (Exception e) {
system.shutdown(); system.terminate();
e.printStackTrace(); e.printStackTrace();
}finally {
system.terminate();
} }
break; break;
case "FORKJOIN": case "FORKJOIN":