azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new dataset

Browse Source
This commit is contained in:
Anil Koyuncu
2018-04-18 18:54:49 +02:00
parent ec0b90a6b0
commit 30f33270c1
5 changed files with 100 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java
+8 -6
View File
@@ -21,13 +21,9 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel; import java.nio.channels.FileChannel;
import java.time.Duration; import java.time.Duration;
import java.util.*; import java.util.*;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis;
/** /**
* Created by anilkoyuncu on 19/03/2018. * Created by anilkoyuncu on 19/03/2018.
*/ */
@@ -39,7 +35,7 @@ public class TreeLoaderClusterL1 {
private static Logger log = LoggerFactory.getLogger(TreeLoaderClusterL1.class); private static Logger log = LoggerFactory.getLogger(TreeLoaderClusterL1.class);
// public static void main(String[] args){ // public static void main(String[] args){
public static void main(String portInner,String serverWait,String port,String inputPath,String level1DB,String level1Path) throws Exception { public static void main(String portInner,String serverWait,String port,String inputPath,String level1DB,String level1Path,String innerTypePrefix) throws Exception {
// String inputPath; // String inputPath;
// String outputPath; // String outputPath;
@@ -81,6 +77,7 @@ public class TreeLoaderClusterL1 {
Stream<File> stream = Arrays.stream(listFiles); Stream<File> stream = Arrays.stream(listFiles);
List<File> dbs = stream List<File> dbs = stream
.filter(x -> x.getName().endsWith(".rdb")) .filter(x -> x.getName().endsWith(".rdb"))
.filter(x-> x.getName().startsWith(innerTypePrefix))
.collect(Collectors.toList()); .collect(Collectors.toList());
for (File db : dbs) { for (File db : dbs) {
String cmdInner = "bash "+inputPath + "/" + "startServer.sh" +" %s %s %s"; String cmdInner = "bash "+inputPath + "/" + "startServer.sh" +" %s %s %s";
@@ -137,12 +134,17 @@ public class TreeLoaderClusterL1 {
innerPool.close(); innerPool.close();
} }
String stopServer = "bash "+level1Path + "/" + "stopServer.sh" +" %s"; String stopServer = "bash "+inputPath + "/" + "stopServer.sh" +" %s";
stopServer = String.format(stopServer,Integer.valueOf(portInner)); stopServer = String.format(stopServer,Integer.valueOf(portInner));
// loadRedis(stopServer,serverWait); // loadRedis(stopServer,serverWait);
cs.runShell(stopServer,serverWait); cs.runShell(stopServer,serverWait);
} }
String stopServer1 = "bash "+inputPath + "/" + "stopServer.sh" +" %s";
stopServer1 = String.format(stopServer1,Integer.valueOf(port));
// loadRedis(stopServer,serverWait);
cs.runShell(stopServer1,serverWait);
// calculatePairsOfClusters(inputPath, outputPath); // calculatePairsOfClusters(inputPath, outputPath);
// mainCompare(inputPath,port,pairsCSVPath,importScript); // mainCompare(inputPath,port,pairsCSVPath,importScript);
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java
+2
View File
@@ -95,6 +95,8 @@ public class FixedViolationHunkParser extends FixedViolationParser {
} }
} }
// public void parseFixPatterns(File prevFile, File revFile, File diffentryFile) { // public void parseFixPatterns(File prevFile, File revFile, File diffentryFile) {
// // GumTree results // // GumTree results
// List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); // List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile);
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster.java
+16 -12
View File
@@ -119,15 +119,18 @@ public class MultiThreadTreeLoaderCluster {
log.info("Load done"); log.info("Load done");
} }
public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner) { public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String serverWait,String type) throws Exception {
CallShell cs = new CallShell();
String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner)); cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner));
edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000"); // edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000");
cs.runShell(cmd1,serverWait);
// String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
// cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port)); cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port));
cs.runShell(cmd2,serverWait);
// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000"); // edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000");
@@ -164,7 +167,8 @@ public class MultiThreadTreeLoaderCluster {
if (size == 0) { if (size == 0) {
String comd = String.format(cmd3,f.getPath(),portInner); String comd = String.format(cmd3,f.getPath(),portInner);
loadRedis(comd); // loadRedis(comd);
cs.runShell(comd);
scan = jedis.scan("0", sc); scan = jedis.scan("0", sc);
size = scan.getResult().size(); size = scan.getResult().size();
@@ -179,7 +183,7 @@ public class MultiThreadTreeLoaderCluster {
//76 //76
scan.getResult().parallelStream() scan.getResult().parallelStream()
.forEach(m -> coreCompare(m, jedisPool, clusterName,outerPool)); .forEach(m -> coreCompare(m, jedisPool, clusterName,outerPool,type));
jedis.save(); jedis.save();
@@ -207,7 +211,7 @@ public class MultiThreadTreeLoaderCluster {
} }
public static Pair<ITree,String> getTree(String firstValue, JedisPool outerPool){ public static Pair<ITree,String> getTree(String firstValue, JedisPool outerPool,String type){
// HierarchicalActionSet actionSet = null; // HierarchicalActionSet actionSet = null;
@@ -245,7 +249,7 @@ public class MultiThreadTreeLoaderCluster {
try { try {
inner = outerPool.getResource(); inner = outerPool.getResource();
String filename = project + "/ActionSetDumps/" + pureFileName + ".txt_" + actionSetPosition; String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition;
String si= inner.get(filename); String si= inner.get(filename);
HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(si); HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(si);
@@ -405,7 +409,7 @@ public class MultiThreadTreeLoaderCluster {
private static void coreCompare(String name , JedisPool jedisPool,String clusterName,JedisPool outerPool) { private static void coreCompare(String name , JedisPool jedisPool,String clusterName,JedisPool outerPool,String type) {
try (Jedis jedis = jedisPool.getResource()) { try (Jedis jedis = jedisPool.getResource()) {
@@ -459,8 +463,8 @@ public class MultiThreadTreeLoaderCluster {
// } // }
try { try {
Pair<ITree, String> oldPair = getTree(firstValue, outerPool); Pair<ITree, String> oldPair = getTree(firstValue, outerPool,type);
Pair<ITree, String> newPair = getTree(secondValue, outerPool); Pair<ITree, String> newPair = getTree(secondValue, outerPool,type);
ITree oldTree = oldPair.getValue0(); ITree oldTree = oldPair.getValue0();
ITree newTree = newPair.getValue0(); ITree newTree = newPair.getValue0();
@@ -631,7 +635,7 @@ orginal calculate pairs, from all dumps of the projects
String line = null; String line = null;
try { try {
FileOutputStream fos = new FileOutputStream(outputPath + "pairs/" +filename+".txt"); FileOutputStream fos = new FileOutputStream(outputPath + "/pairs/" +filename+".txt");
DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos));
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java
+62 -30
View File
@@ -161,17 +161,18 @@ public class MultiThreadTreeLoaderCluster3 {
} }
// public static void mainCompare(String inputPath,String port,String pairsCSVPath,String importScript) { // public static void mainCompare(String inputPath,String port,String pairsCSVPath,String importScript) {
public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner) { public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String serverWait, String type) throws Exception {
CallShell cs = new CallShell();
String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner)); cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner));
edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000"); // edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000");
cs.runShell(cmd1,serverWait);
String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port)); cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port));
edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000"); // edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000");
cs.runShell(cmd2,serverWait);
String cmd3; String cmd3;
cmd3 = "bash " + importScript +" %s %s"; cmd3 = "bash " + importScript +" %s %s";
@@ -207,8 +208,9 @@ public class MultiThreadTreeLoaderCluster3 {
if (size == 0) { if (size == 0) {
String comd = String.format(cmd3, f.getPath(), portInner); String comd = String.format(cmd3, f.getPath(), portInner);
edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster. cs.runShell(comd);
loadRedis(comd); // edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster.
// loadRedis(comd);
scan = jedis.scan("0", sc); scan = jedis.scan("0", sc);
size = scan.getResult().size(); size = scan.getResult().size();
@@ -221,7 +223,7 @@ public class MultiThreadTreeLoaderCluster3 {
scan.getResult().parallelStream() scan.getResult().parallelStream()
.forEach(m -> coreCompare(m, jedisPool, clusterName, outerPool)); .forEach(m -> coreCompare(m, jedisPool, clusterName, outerPool,type));
jedis.save(); jedis.save();
@@ -242,7 +244,7 @@ public class MultiThreadTreeLoaderCluster3 {
public static ITree getTree(String firstValue, JedisPool outerPool){ public static ITree getTree(String firstValue, JedisPool outerPool,String type){
// String gumTreeInput = "/Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/"; // String gumTreeInput = "/Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/";
String[] split2 = firstValue.split("/"); String[] split2 = firstValue.split("/");
String cluster = split2[1]; String cluster = split2[1];
@@ -268,7 +270,7 @@ public class MultiThreadTreeLoaderCluster3 {
try { try {
inner = outerPool.getResource(); inner = outerPool.getResource();
String fn = project + "/ActionSetDumps/" + s + ".txt_" + actionSetPosition; String fn = project + "/"+type+"/" + s + ".txt_" + actionSetPosition;
String si= inner.get(fn); String si= inner.get(fn);
HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(si); HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(si);
@@ -443,6 +445,36 @@ public class MultiThreadTreeLoaderCluster3 {
addToken = true; addToken = true;
} }
}else if(sType.equals("7")){//assignment
m.add(hours.trim().split("=")[0]);
m.add(to.trim().split("=")[0]);
addToken = true;
}else if(sType.equals("43") && oldDescendant.getParent() != null && oldDescendant.getParent().getParent() != null){// catch clause
if(oldDescendant.getParent().getType() == 44 && oldDescendant.getParent().getParent().getType() == 12){
m.add(hours.trim());
m.add(to.trim());
addToken = true;
}
}else if(sType.equals("59") && oldDescendant.getChildren().size()==1 && oldDescendant.getHeight()==1 ){
if( oldDescendant.getChild(0).getType() == 34 || oldDescendant.getChild(0).getType() == 45) {
m.add(hours.trim().split("=")[0]);
m.add(to.trim().split("=")[0]);
addToken = true;
}
}else if(sType.equals("42") && oldDescendant.getParent() != null && oldDescendant.getParent().getType() == 27 ){//infix varuable change
m.add(hours.trim());
m.add(to.trim());
addToken = true;
}else if(sType.equals("-1") && oldDescendant.getChildren().size()==0 && oldDescendant.getHeight()==0 ){//operator
m.add(hours.trim());
m.add(to.trim());
addToken = true;
}else if(sType.equals("83") && oldDescendant.getChildren().size()==0 && oldDescendant.getHeight()==0 ){//modifier
m.add(hours.trim());
m.add(to.trim());
addToken = true;
}else{ }else{
m.add(hours.trim()); m.add(hours.trim());
m.add(to.trim()); m.add(to.trim());
@@ -504,23 +536,23 @@ public class MultiThreadTreeLoaderCluster3 {
oldTokens.add(methodName); oldTokens.add(methodName);
}else if (s.startsWith("MethodName:")){ }else if (s.startsWith("MethodName:")){
String methodName = s.split(":")[1]; String methodName = s.split(":")[1];
// ITree parent = oldDescendant.getParent(); ITree parent = oldDescendant.getParent();
//
// if(parent!= null && parent.getType() == 32 && !alreadyAddParentMethodName){ //parent is method invocation statement if(parent!= null && parent.getType() == 32 && !alreadyAddParentMethodName){ //parent is method invocation statement
// String parentLabel = parent.getLabel(); String parentLabel = parent.getLabel();
// String[] pns = parentLabel.split("\\." + methodName); String[] pns = parentLabel.split("\\." + methodName);
// if(pns.length > 1) { if(pns.length > 1) {
// String parentName = pns[0]; String parentName = pns[0];
// String[] parentNameSplit = parentName.split("@@"); String[] parentNameSplit = parentName.split("@@");
// if (parentNameSplit.length > 1) { if (parentNameSplit.length > 1) {
// String parentMethodName = parentNameSplit[1]; String parentMethodName = parentNameSplit[1];
// String s1 = parentMethodName.split("@TO@")[0]; String s1 = parentMethodName.split("@TO@")[0];
// oldTokens.add(s1.trim()); oldTokens.add(s1.trim());
// alreadyAddParentMethodName = true; alreadyAddParentMethodName = true;
// } }
// } }
//
// } }
oldTokens.add(methodName); oldTokens.add(methodName);
}else if( sType.equals("59") || sType.equals("43")|| sType.equals("53") || sType.equals("7") || sType.equals("27") || sType.equals("83") || sType.equals("44") ||sType.equals("78") || sType.equals("41") || addToken){ }else if( sType.equals("59") || sType.equals("43")|| sType.equals("53") || sType.equals("7") || sType.equals("27") || sType.equals("83") || sType.equals("44") ||sType.equals("78") || sType.equals("41") || addToken){
// if(sType.equals("27") || sType.equals("83") || sType.equals("44")){ // if(sType.equals("27") || sType.equals("83") || sType.equals("44")){
@@ -570,7 +602,7 @@ public class MultiThreadTreeLoaderCluster3 {
return oldTokens; return oldTokens;
} }
private static void coreCompare(String name , JedisPool jedisPool,String clusterName,JedisPool outerPool) { private static void coreCompare(String name , JedisPool jedisPool,String clusterName,JedisPool outerPool,String type) {
try (Jedis jedis = jedisPool.getResource()) { try (Jedis jedis = jedisPool.getResource()) {
@@ -621,8 +653,8 @@ public class MultiThreadTreeLoaderCluster3 {
// } // }
try { try {
ITree oldTree = getTree(firstValue,outerPool); ITree oldTree = getTree(firstValue,outerPool,type);
ITree newTree = getTree(secondValue,outerPool); ITree newTree = getTree(secondValue,outerPool,type);
if(oldTree == null || newTree == null) { if(oldTree == null || newTree == null) {
jedis.select(0); jedis.select(0);
src/main/java/edu/lu/uni/serval/Launcher.java
+12 -7
View File
@@ -46,11 +46,11 @@ public class Launcher {
// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs"; // inputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs";
// gumInput = "/Users/anilkoyuncu/bugStudy/dataset/Defects4J/"; // gumInput = "/Users/anilkoyuncu/bugStudy/dataset/Defects4J/";
portInner = "6380"; portInner = "6380";
serverWait = "50000"; serverWait = "5000";
chunkName = "Bug13April.txt.csv.rdb"; chunkName = "Bug13April.txt.csv.rdb";
// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; // dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
numOfWorkers = "10"; numOfWorkers = "10";
jobType = "AKKA"; jobType = "LEVEL1DB";
port = "6399"; port = "6399";
// pairsPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImportDefects4J"; // pairsPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImportDefects4J";
// gumOutput = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputDefects4J"; // gumOutput = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputDefects4J";
@@ -97,23 +97,28 @@ public class Launcher {
AkkaTreeLoader.main(portInner, serverWait, dbDir, chunk +"MOV"+".txt.csv.rdb", port, "MOV"+dumpsName); AkkaTreeLoader.main(portInner, serverWait, dbDir, chunk +"MOV"+".txt.csv.rdb", port, "MOV"+dumpsName);
break; break;
case "LEVEL1DB": case "LEVEL1DB":
TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-BugsDotJar.rdb", dbDir + "/level1-BugsDotJar/"); TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ "UPD"+".rdb", dbDir + "/level1-Bug13April/",pjName + "UPD");
TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ "INS"+".rdb", dbDir + "/level1-Bug13April/",pjName + "INS");
TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ "DEL"+".rdb", dbDir + "/level1-Bug13April/",pjName + "DEL");
TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ "MOV"+".rdb", dbDir + "/level1-Bug13April/",pjName + "MOV");
break; break;
//CALC python abstractPatch.py to from cluster folder //CALC python abstractPatch.py to from cluster folder
case "L2CALCPAIRS": case "L2CALCPAIRS":
// MultiThreadTreeLoaderCluster.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterDefect4J","/Users/anilkoyuncu/bugStudy/dataset/"); // MultiThreadTreeLoaderCluster.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterDefect4J","/Users/anilkoyuncu/bugStudy/dataset/");
MultiThreadTreeLoaderCluster.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/cluster", datasetPath); MultiThreadTreeLoaderCluster.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterBug13April", datasetPath);
break; break;
case "L2PAIRDB": case "L2PAIRDB":
// MultiThreadTreeLoaderCluster.mainCompare("6300","/Users/anilkoyuncu/bugStudy/dataset/pairs-csv","/Users/anilkoyuncu/bugStudy/dataset/redisSingleImport.sh",dbDir,"clusterl1-d4j.rdb",dumpsName,"6301"); // MultiThreadTreeLoaderCluster.mainCompare("6300","/Users/anilkoyuncu/bugStudy/dataset/pairs-csv","/Users/anilkoyuncu/bugStudy/dataset/redisSingleImport.sh",dbDir,"clusterl1-d4j.rdb",dumpsName,"6301");
MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath + "/pairs-csv", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-13april.rdb", dumpsName, "6301"); MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath + "/pairs-csv", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-13april.rdb", "UPD"+dumpsName, "6301",serverWait,"UPD");
break; break;
//CALC via python
case "L3CALCPAIRS": case "L3CALCPAIRS":
// MultiThreadTreeLoaderCluster3.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterDefect4J-2l",datasetPath); // MultiThreadTreeLoaderCluster3.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterDefect4J-2l",datasetPath);
MultiThreadTreeLoaderCluster3.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/cluster-2l", datasetPath); MultiThreadTreeLoaderCluster3.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterBug13April-2l", datasetPath);
break; break;
case "L3PAIRDB": case "L3PAIRDB":
MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath + "/pairs-2l-csv", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-13april.rdb", dumpsName, "6301"); MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath + "/pairs-2l-csv", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-13april.rdb", "UPD"+dumpsName, "6301",serverWait,"UPD");
break; break;
} }
} catch (Exception e) { } catch (Exception e) {