azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

launcher

Browse Source
This commit is contained in:
Anil Koyuncu
2018-04-14 19:07:53 +02:00
parent ae0c548e20
commit c03061db69
9 changed files with 677 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pom.xml
+1 -1
View File
@@ -108,7 +108,7 @@
<archive>
<manifest>
<mainClass>
edu.lu.uni.serval.FixPatternParser.cluster.ImportPairs2DB
edu.lu.uni.serval.Launcher
</mainClass>
</manifest>
</archive>
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java
+41 -41
View File
@@ -92,48 +92,48 @@ public class AkkaTreeLoader {
log.info("Load done");
}
public static void main(String[] args) {
// public static void main(String[] args) {
public static void main(String inputPath,String portInner,String serverWait,String dbDir,String chunkName,String numOfWorkers,String port){
String inputPath;
// String outputPath;
String port;
String portInner;
// String pairsCSVPath;
String importScript;
// String pairsCompletedPath;
String serverWait;
// String option;
String dbDir;
String chunkName;
String numOfWorkers;
if (args.length > 0) {
inputPath = args[0];
portInner = args[1];
serverWait = args[2];
// option = args[4];
chunkName = args[3];
numOfWorkers = args[4];
dbDir = args[5];
port = args[6];
// pairsCSVPath = args[3];
// importScript = args[4];
// pairsCompletedPath = args[3];
} else {
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/";
port = "6399";
portInner = "6380";
serverWait = "10000";
// option = "COMP";
// pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test";
// importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh";
// pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed";
chunkName ="chunk3.rdb";
dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
numOfWorkers = "1";
}
// String inputPath;
//// String outputPath;
// String port;
// String portInner;
//// String pairsCSVPath;
// String importScript;
//// String pairsCompletedPath;
// String serverWait;
//// String option;
// String dbDir;
// String chunkName;
// String numOfWorkers;
// if (args.length > 0) {
// inputPath = args[0];
// portInner = args[1];
// serverWait = args[2];
//// option = args[4];
// chunkName = args[3];
// numOfWorkers = args[4];
// dbDir = args[5];
// port = args[6];
//// pairsCSVPath = args[3];
//// importScript = args[4];
//// pairsCompletedPath = args[3];
// } else {
// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
//// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/";
// port = "6399";
// portInner = "6380";
// serverWait = "10000";
//// option = "COMP";
//// pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test";
//// importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh";
//// pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed";
// chunkName ="chunk3.rdb";
// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
// numOfWorkers = "1";
//
// }
String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \nnumOfWorks %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,numOfWorkers,dbDir);
log.info(parameters);
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java
+32 -32
View File
@@ -29,39 +29,39 @@ import static edu.lu.uni.serval.FixPatternParser.cluster.TreeLoaderClusterL1.poo
*/
public class CalculatePairs {
private static Logger log = LoggerFactory.getLogger(CalculatePairs.class);
public static void main(String[] args) {
// public static void main(String[] args) {
public static void main(String inputPath,String portInner,String serverWait,String dbDir,String chunkName,String numOfWorkers,String port,String outputPath){
String inputPath;
String port;
String portInner;
String serverWait;
String dbDir;
String chunkName;
String outputPath;
if (args.length > 0) {
inputPath = args[0];
portInner = args[1];
serverWait = args[2];
chunkName = args[3];
dbDir = args[5];
port = args[6];
outputPath = args[7];
} else {
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
port = "6399";
portInner = "6380";
serverWait = "10000";
outputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImport";
chunkName ="chunk";
dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
}
// String inputPath;
// String port;
// String portInner;
// String serverWait;
// String dbDir;
// String chunkName;
// String outputPath;
//
// if (args.length > 0) {
// inputPath = args[0];
// portInner = args[1];
// serverWait = args[2];
// chunkName = args[3];
//
// dbDir = args[5];
// port = args[6];
// outputPath = args[7];
//
// } else {
// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
//
// port = "6399";
// portInner = "6380";
// serverWait = "10000";
// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImport";
// chunkName ="chunk";
// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
//
//
// }
String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,dbDir);
log.info(parameters);
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java
+25 -23
View File
@@ -19,31 +19,33 @@ import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedi
*/
public class ImportPairs2DB {
private static Logger log = LoggerFactory.getLogger(ImportPairs2DB.class);
public static void main(String[] args) {
// public static void main(String[] args) {
String inputPath;
String portInner;
String serverWait;
String dbDir;
String chunkName;
String numOfWorkers;
if (args.length > 0) {
inputPath = args[0];
portInner = args[1];
serverWait = args[2];
chunkName = args[3];
numOfWorkers = args[4];
dbDir = args[5];
} else {
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs";
portInner = "6380";
serverWait = "10000";
chunkName ="dumps.rdb";
dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
numOfWorkers = "1";
}
public static void main(String inputPath,String portInner,String serverWait,String dbDir,String chunkName,String numOfWorkers){
// String inputPath;
// String portInner;
// String serverWait;
// String dbDir;
// String chunkName;
// String numOfWorkers;
// if (args.length > 0) {
// inputPath = args[0];
// portInner = args[1];
// serverWait = args[2];
// chunkName = args[3];
// numOfWorkers = args[4];
// dbDir = args[5];
// } else {
// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs";
// portInner = "6380";
// serverWait = "10000";
// chunkName ="dumps.rdb";
// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
// numOfWorkers = "1";
// }
String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \nnumOfWorks %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,numOfWorkers,dbDir);
log.info(parameters);
File folder = new File(inputPath);
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java
+24 -24
View File
@@ -25,31 +25,31 @@ public class StoreFile {
private static Logger log = LoggerFactory.getLogger(StoreFile.class);
public static void main(String[] args) {
String inputPath;
String portInner;
String serverWait;
String dbDir;
String chunkName;
String numOfWorkers;
if (args.length > 0) {
inputPath = args[0];
portInner = args[1];
serverWait = args[2];
chunkName = args[3];
numOfWorkers = args[4];
dbDir = args[5];
} else {
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
portInner = "6399";
serverWait = "10000";
chunkName ="dumps.rdb";
dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
numOfWorkers = "1";
}
// public static void main(String[] args) {
public static void main(String inputPath,String portInner,String serverWait,String dbDir,String chunkName,String numOfWorkers){
// String inputPath;
// String portInner;
// String serverWait;
// String dbDir;
// String chunkName;
// String numOfWorkers;
// if (args.length > 0) {
// inputPath = args[0];
// portInner = args[1];
// serverWait = args[2];
// chunkName = args[3];
// numOfWorkers = args[4];
// dbDir = args[5];
// } else {
// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
// portInner = "6399";
// serverWait = "10000";
// chunkName ="dumps.rdb";
// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
// numOfWorkers = "1";
// }
String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \nnumOfWorks %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,numOfWorkers,dbDir);
log.info(parameters);
String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner));
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java
+128 -163
View File
@@ -12,6 +12,7 @@ import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.utils.ListSorter;
import org.apache.commons.lang3.StringUtils;
import org.javatuples.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -96,7 +97,7 @@ public class MultiThreadTreeLoaderCluster3 {
cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
cmd = String.format(cmd, dbDir,"dumps.rdb",Integer.valueOf("6399"));
edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd,"10000");
edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd,"1000");
mainCompare(inputPath,port,pairsCSVPath,importScript);
// calculatePairs(inputPath, outputPath);
@@ -176,8 +177,8 @@ public class MultiThreadTreeLoaderCluster3 {
.collect(Collectors.toList());
for (File f:folders){
if(f.getName().startsWith("cluster7_0")) {
//36_0,119_0,4_0
// if(f.getName().startsWith("cluster1_0")) {
try (Jedis jedis = jedisPool.getResource()) {
@@ -217,7 +218,7 @@ public class MultiThreadTreeLoaderCluster3 {
}
// }
@@ -245,48 +246,9 @@ public class MultiThreadTreeLoaderCluster3 {
}
// File folder = new File("/Users/anilkoyuncu/bugStudy/code/python/clusterDumps/"+cluster + "/" + s + ".txt_" + actionSetPosition);
//
//
// ITree tree = null;
// try {
// FileInputStream fi = new FileInputStream(folder);
// ObjectInputStream oi = new ObjectInputStream(fi);
// tree = (ITree) oi.readObject();
// oi.close();
// fi.close();
//
//
// } catch (FileNotFoundException e) {
// log.error("File not found");
// e.printStackTrace();
// } catch (IOException e) {
// log.error("Error initializing stream");
// e.printStackTrace();
// } catch (ClassNotFoundException e) {
// // TODO Auto-generated catch block
// e.printStackTrace();
// }
// return tree;
ITree tree = null;
Jedis inner = null;
// String[] split2 = firstValue.split("/");
// String cluster = split2[1];
//
// String[] split1 = folders.get(0).getName().split(".txt_");
// String s = split1[0];
// String[] splitPJ = split1[1].split("_");
// String project = splitPJ[1];
// String actionSetPosition = splitPJ[0];
// File folder = new File("/Users/anilkoyuncu/bugStudy/code/python/cluster/"+cluster);
// File folder = new File("/Users/anilkoyuncu/bugStudy/code/python/clusterDumps/"+cluster + "/" + s + ".txt_" + actionSetPosition);
// File[] listOfFiles = folder.listFiles();
// Stream<File> stream = Arrays.stream(listOfFiles);
// List<File> folders = stream
// .filter(x -> !x.getName().startsWith(".") && x.getName().startsWith(split2[2]))
// .collect(Collectors.toList());
@@ -296,15 +258,6 @@ public class MultiThreadTreeLoaderCluster3 {
String si= inner.get(fn);
HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(si);
// ITree newTree = ((Update)actionSet.getAction()).getNewNode();
// ITree oldTree = ((Update)actionSet.getAction()).getNode();
//
// Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
// m.match();
// ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings());
// ag.generate();
// List<Action> actions = ag.getActions();
// log.info(actions.toString());
ITree parent = null;
ITree children =null;
@@ -322,46 +275,10 @@ public class MultiThreadTreeLoaderCluster3 {
inner.close();
}
}
// Pair<ITree, String> pair = new Pair<>(tree,project);
return tree;
// File[] listOfFiles = folder.listFiles();
// Stream<File> stream = Arrays.stream(listOfFiles);
// List<File> folders = stream
// .filter(x -> !x.getName().startsWith(".") && x.getName().startsWith(split2[2]))
// .collect(Collectors.toList());
//
//
//
//// String[] split1 = folders.get(0).getName().split(".txt_");
//// String s = split1[0];
//// String[] splitPJ = split1[1].split("_");
//// String project = splitPJ[1];
//// String actionSetPosition = splitPJ[0];
//
// File prevFile = new File(gumTreeInput + project+ "/" + "prevFiles/prev_" + s + ".java");// previous file
// File revFile = new File(gumTreeInput + project+ "/" + "revFiles/" + s + ".java");//rev file
//
// List<HierarchicalActionSet> actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile);
//
// HierarchicalActionSet actionSet = actionSets.get(Integer.valueOf(actionSetPosition));
//// for (HierarchicalActionSet actionSet : actionSets) {
//
// ITree test = getActionTree(actionSet);
// test.getLabel();
// for (ITree descendant : test.getDescendants()) {
//// descendant.setLabel("");
// if(!(descendant.getType() == 100 || descendant.getType() == 101 || descendant.getType() == 102 || descendant.getType() == 103)){
// descendant.setType(104);
// }
// }
// test.getDescendants();
// test.setParent(null);
//// }
//// }
//
// Pair<ITree, String> pair = new Pair<>(test,project);
// return pair;
}
public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){
@@ -436,22 +353,14 @@ public class MultiThreadTreeLoaderCluster3 {
}
private static List<String> getNames(ITree oldTree, List<String> oldTokens){
// if((oldTree.getType() == 42 && oldTree.getLabel().startsWith("Name:")) || oldTree.getType() == 42 && oldTree.getParent().getType() == 59 || oldTree.getType() == 43 || (oldTree.getType() == 41 && oldTree.getLabel().startsWith("SimpleName:")) ){
//
// oldTokens.add(oldTree.getLabel());
//
// }
// for (ITree oldDescendant : oldTree.getDescendants()) {
// if ((oldDescendant.getType() == 42 && oldDescendant.getLabel().startsWith("Name:") ) || oldDescendant.getType() == 42 && oldDescendant.getParent().getType() == 59 ||oldDescendant.getType() == 43 || (oldDescendant.getType() == 41 && oldDescendant.getLabel().startsWith("SimpleName:"))){
//
// oldTokens.add(oldDescendant.getLabel());
//
// }
// }
List<ITree> descendants = oldTree.getDescendants();
descendants.add(0,oldTree);
boolean upd=false;
for (ITree oldDescendant : descendants) {
boolean addToken = false;
int type = oldDescendant.getType();
String sType = String.valueOf(type);
@@ -460,18 +369,18 @@ public class MultiThreadTreeLoaderCluster3 {
(sType.equals("32") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1) ||
(sType.equals("59") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1) ||
(sType.equals("43") && oldDescendant.getHeight() == 0 && oldDescendant.getChildren().size() == 0) ||
(sType.equals("14") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1) ||
(sType.equals("53") )|| //&& oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1) ||
(sType.equals("7") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1) ||
(sType.equals("27") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1)
(sType.equals("41") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1)
){
int depth = oldDescendant.getChildren().size();
// int depth = oldDescendant.getChildren().size();
String label = oldDescendant.getLabel();
if(sType.equals("32") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1){
log.info("");
}
// if(sType.equals("32") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1){
// log.info("");
// }
label = label.split("@AT@")[0];
@@ -480,15 +389,50 @@ public class MultiThreadTreeLoaderCluster3 {
List<String> m = new ArrayList<String>();
if(label.startsWith("UPD")){
upd = true;
String timeRegex = "@@(.*)@TO@(.*)";
Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL);
java.util.regex.Matcher matcher = pattern.matcher(split[1]);
java.util.regex.Matcher matcher;
if(sType.equals("53")){
String timeRegex = ".*@@(ClassInstanceCreation:new [a-zA-Z0-9]+).*@TO@\\s(ClassInstanceCreation:new [a-zA-Z0-9]+).*";
Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL);
matcher= pattern.matcher(split[1]);
}else {
String timeRegex = "@@(.*)@TO@(.*)";
Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL);
matcher = pattern.matcher(split[1]);
}
if (matcher.matches()) {
String hours = matcher.group(1);
String to = matcher.group(2);
m.add(hours.trim());
m.add(to.trim());
if(sType.equals("31")){
String commonPrefix = StringUtils.getCommonPrefix(hours.trim(), to.trim());
if(commonPrefix.isEmpty()){
log.info("PREFIX EMPTY");
}else {
String s = hours.trim().replace(commonPrefix, "");
String s1 = to.trim().replace(commonPrefix, "");
String[] split1 = s.split(",");
String[] split3 = s1.split(",");
Set<String> set = new TreeSet<>();
for (String s2 : split1) {
if(!s2.isEmpty()) {
set.add(s2.trim());
}
}
for (String s2 : split3) {
if(!s2.isEmpty()) {
set.add(s2.trim());
}
}
List<String> list = set.stream().collect(Collectors.toList());
m.addAll(list);
addToken = true;
}
}else{
m.add(hours.trim());
m.add(to.trim());
}
}
}else if(label.startsWith("INS") && upd == false){
@@ -521,25 +465,67 @@ public class MultiThreadTreeLoaderCluster3 {
}
}
}
boolean alreadyAddParentMethodName = false;
for (String s : m) {
// if(s.isEmpty()){
// continue;
// }
//TODO remove 44
if(s.startsWith("SimpleName:") || s.startsWith("Name:")){
oldTokens.add(s);
}else if (s.startsWith("MethodName:")){
String methodName = s.split(":")[1];
oldTokens.add(methodName);
}else if( sType.equals("59") || sType.equals("43")|| sType.equals("14") || sType.equals("7") || sType.equals("27") || sType.equals("83") || sType.equals("44")){
if(sType.equals("27") || sType.equals("83") || sType.equals("44")){
oldTokens.add(s);
}
else {
String s1 = s.split("=")[0];
oldTokens.add(s1);
}else if (s.startsWith("MethodName:")){
String methodName = s.split(":")[1];
// ITree parent = oldDescendant.getParent();
//
// if(parent!= null && parent.getType() == 32 && !alreadyAddParentMethodName){ //parent is method invocation statement
// String parentLabel = parent.getLabel();
// String[] pns = parentLabel.split("\\." + methodName);
// if(pns.length > 1) {
// String parentName = pns[0];
// String[] parentNameSplit = parentName.split("@@");
// if (parentNameSplit.length > 1) {
// String parentMethodName = parentNameSplit[1];
// String s1 = parentMethodName.split("@TO@")[0];
// oldTokens.add(s1.trim());
// alreadyAddParentMethodName = true;
// }
// }
//
// }
oldTokens.add(methodName);
}else if( sType.equals("59") || sType.equals("43")|| sType.equals("53") || sType.equals("7") || sType.equals("27") || sType.equals("83") || sType.equals("44") ||sType.equals("78") || sType.equals("41") || addToken){
// if(sType.equals("27") || sType.equals("83") || sType.equals("44")){
// String parentLabel = oldDescendant.getParent().getLabel();
// List<String> parentM = new ArrayList<String>();
// if(parentLabel.startsWith("UPD")) {
// upd = true;
// String timeRegex = "@@(.*)@TO@(.*)";
// Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL);
// java.util.regex.Matcher matcher = pattern.matcher(split[1]);
//
// if (matcher.matches()) {
// String hours = matcher.group(1);
// String to = matcher.group(2);
// parentM.add(hours.trim());
// parentM.add(to.trim());
//
// }
// }
//
//
// oldTokens.add(s);
// }else
if(sType.equals("53") || sType.equals("78") || addToken){//sType.equals("41") ||
oldTokens.add(s);
}
// else {
// String s1 = s.split("=")[0];
// oldTokens.add(s1);
// }
}
// else
// if(oldTokens.size() < 2){
@@ -550,9 +536,9 @@ public class MultiThreadTreeLoaderCluster3 {
}
}
if (oldTokens.size() == 0){
log.info("dur bakalim nereye!???");
}
// if (oldTokens.size() == 0 || oldTokens.size() > 3) {// && (oldTree.getType() != 41 && oldTree.getType() != 21 && oldTree.getType() !=17 && oldTree.getType()!=60 && oldTree.getType() != 46)){
// log.info("dur bakalim nereye!???");
// }
return oldTokens;
}
@@ -580,9 +566,9 @@ public class MultiThreadTreeLoaderCluster3 {
String firstValue = resultMap.get("0");
String secondValue = resultMap.get("1");
if(firstValue.equals("7/0/7af9e0_e674f1_spring-social-web#src#main#java#org#springframework#social#connect#web#ConnectController.txt_0_SOCIAL") || secondValue.equals("7/0/7af9e0_e674f1_spring-social-web#src#main#java#org#springframework#social#connect#web#ConnectController.txt_0_SOCIAL")){
log.info("");
}
// if(firstValue.equals("7/0/7af9e0_e674f1_spring-social-web#src#main#java#org#springframework#social#connect#web#ConnectController.txt_0_SOCIAL") || secondValue.equals("7/0/7af9e0_e674f1_spring-social-web#src#main#java#org#springframework#social#connect#web#ConnectController.txt_0_SOCIAL")){
// log.info("");
// }
///35/1/22b5f7_84bf27_ui#org.eclipse.pde.runtime#src#org#eclipse#pde#internal#runtime#registry#RegistryBrowserLabelProvider.txt_2_PDE
@@ -635,6 +621,10 @@ public class MultiThreadTreeLoaderCluster3 {
oldTokens = getNames(oldTree,oldTokens);
newTokens = getNames(newTree,newTokens);
if(oldTokens.size() == 0 || newTokens.size() == 0){
log.error("Cluster {} has no tokens on pair {}",clusterName , name);
}
// Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree);
// m.match();
CharSequence[] oldSequences = oldTokens.toArray(new CharSequence[oldTokens.size()]);
@@ -648,17 +638,21 @@ public class MultiThreadTreeLoaderCluster3 {
}
}else{
overallSimi = Double.valueOf(0);
if(oldSequences.length != 0) {
log.info("ERROR");
}
// if(oldSequences.length != 0) {
// log.info("ERROR");
// }
}
if(overallSimi.equals(1.0)){
int retval = Double.compare(overallSimi, Double.valueOf(0.8));
if(retval >= 0){
// log.info("YES");
// log.info(name);
// log.info(firstValue);
// log.info(secondValue);
// log.info("************");
if(!overallSimi.equals(1.0)){
log.info("");
}
String matchKey = "match-"+clusterName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j);
String result = firstValue + "," + secondValue + ","+String.join(",", oldTokens);
jedis.select(1);
@@ -670,35 +664,6 @@ public class MultiThreadTreeLoaderCluster3 {
// ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings());
// ag.generate();
// List<Action> actions = ag.getActions();
//
// String resultKey = "result_" + (String.valueOf(i)) + "_" + String.valueOf(j);
// double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree);
// String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1);
// double diceSimilarity1 = m.diceSimilarity(oldTree, newTree);
// String diceSimilarity = String.format("%1.2f", diceSimilarity1);
// double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree);
// String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1);
//
// String editDistance = String.valueOf(actions.size());
// jedis.select(1);
// String result = resultMap.get("0") + "," + oldProject +"," + resultMap.get("1") + "," +newProject+ "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance;
//// jedis.set(resultKey, result);
//
// if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0)
// || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) {
// String matchKey = "match-"+clusterName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j);
// jedis.select(1);
// jedis.set(matchKey, result);
// }
// jedis.select(0);
//
//// log.info("Completed " + resultKey);
}catch (Exception e){
log.error(e.toString() + " {}",(name));
src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParserSingleFile.java
+363
View File
@@ -0,0 +1,363 @@
package edu.lu.uni.serval.FixPatternParser.violations;
import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import edu.lu.uni.serval.MultipleThreadsParser.MessageFile;
import edu.lu.uni.serval.MultipleThreadsParser.ParseFixPatternActor;
import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage;
import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class TestHunkParserSingleFile {
private static Logger log = LoggerFactory.getLogger(TestHunkParserSingleFile.class);
public static void main(String[] args) {
// input data
// String rootPath = "/Users/anilkoyuncu/bugStudy";
String inputPath;
String outputPath;
if(args.length > 0){
inputPath = args[1];
outputPath = args[0];
}else{
// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeInputBug4";
inputPath = "/Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4";
// outputPath = "/Users/anilkoyuncu/bugStudy/code/python/GumTreeOutput2/";
outputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputSingle";
}
//5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.txt_1_CAMEL
File folder = new File(inputPath);
File[] listOfFiles = folder.listFiles();
Stream<File> stream = Arrays.stream(listOfFiles);
List<File> folders = stream
.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
// List<File> targetList = new ArrayList<File>();
// for (File f:folders){
// for(File f1 :f.listFiles()){
// if (!f1.getName().startsWith(".")){
// targetList.add(f1);
// }
// }
// }
// List<String> pjList = Arrays.asList("DATAJPA","ZXing","PDE","SWS","SWT", "SWF", "COLLECTIONS", "JDT");
List<String> files = new ArrayList<String>();
// files.add("5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.java");
files.add("49821e_3cc787_components#camel-infinispan#src#main#java#org#apache#camel#component#infinispan#InfinispanConstants.java");
for(String f : files){
String pjName = "CAMEL";
// for (File target : folders) {
// String pjName = target.getName();
// if (!pjList.contains(pjName)){
// continue;
// }
// final List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
MessageFile messageFile = getMessageFile(inputPath + "/" + pjName +"/", f);
List<MessageFile> msgFiles = new ArrayList<>();
msgFiles.add(messageFile);
String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/";
final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list";
final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list";
final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list";
final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv";
final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list";
FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ActionSetDumps");
FileHelper.deleteDirectory(editScriptsFilePath);
FileHelper.deleteDirectory(patchesSourceCodeFilePath);
FileHelper.deleteDirectory(buggyTokensFilePath);
FileHelper.deleteDirectory(editScriptSizesFilePath);
FileHelper.deleteDirectory(alarmTypesFilePath);
StringBuilder astEditScripts = new StringBuilder();
StringBuilder tokens = new StringBuilder();
StringBuilder sizes = new StringBuilder();
StringBuilder patches = new StringBuilder();
StringBuilder alarmTypes = new StringBuilder();
// int a = 0;
ActorSystem system = null;
ActorRef parsingActor = null;
final WorkMessage msg = new WorkMessage(0, msgFiles);
try {
log.info("Akka begins...");
system = ActorSystem.create("Mining-FixPattern-System");
parsingActor = system.actorOf(ParseFixPatternActor.props(1, editScriptsFilePath,
patchesSourceCodeFilePath, buggyTokensFilePath, editScriptSizesFilePath), "mine-fix-pattern-actor");
parsingActor.tell(msg, ActorRef.noSender());
} catch (Exception e) {
system.shutdown();
e.printStackTrace();
}
// int counter = 0;
// for (MessageFile msgFile : msgFiles) {
// FixedViolationHunkParser parser = new FixedViolationHunkParser();
//
// final ExecutorService executor = Executors.newSingleThreadExecutor();
// // schedule the work
// final Future<?> future = executor.submit(new RunnableParser(msgFile.getPrevFile(),
// msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser));
// try {
// // where we wait for task to complete
// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS);
// String editScripts = parser.getAstEditScripts();
// if (!editScripts.equals("")) {
// astEditScripts.append(editScripts);
// tokens.append(parser.getTokensOfSourceCode());
// sizes.append(parser.getSizes());
// patches.append(parser.getPatchesSourceCode());
// alarmTypes.append(parser.getAlarmTypes());
//
// a++;
// if (a % 100 == 0) {
// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true);
// FileHelper.outputToFile(buggyTokensFilePath, tokens, true);
// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true);
// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true);
// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true);
// astEditScripts.setLength(0);
// tokens.setLength(0);
// sizes.setLength(0);
// patches.setLength(0);
// alarmTypes.setLength(0);
// System.out.println("Finish of parsing " + a + " files......");
// }
// }
// } catch (TimeoutException e) {
// err.println("task timed out");
// future.cancel(true /* mayInterruptIfRunning */);
// } catch (InterruptedException e) {
// err.println("task interrupted");
// } catch (ExecutionException e) {
// err.println("task aborted");
// } finally {
// executor.shutdownNow();
// }
// }
FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true);
FileHelper.outputToFile(buggyTokensFilePath, tokens, true);
FileHelper.outputToFile(editScriptSizesFilePath, sizes, true);
FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true);
FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true);
astEditScripts.setLength(0);
tokens.setLength(0);
sizes.setLength(0);
patches.setLength(0);
alarmTypes.setLength(0);
// classifyByAlarmTypes();
}
}
private static List<MessageFile> getMessageFiles(String gumTreeInput) {
String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile
File revFilesPath = new File(inputPath + "revFiles/");
File[] revFiles = revFilesPath.listFiles(); // project folders
List<MessageFile> msgFiles = new ArrayList<>();
// gumTreeInput = /Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/AMQP/
// fileName = 01534a_df5570_spring-rabbit#src#test#java#org#springframework#amqp#rabbit#listener#LocallyTransactedTests.java
if (revFiles.length >= 0) {
for (File revFile : revFiles) {
// if (revFile.getName().endsWith(".java")) {
String fileName = revFile.getName();
File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
fileName = fileName.replace(".java", ".txt");
File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file
File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
msgFile.setPositionFile(positionFile);
msgFiles.add(msgFile);
// }
}
return msgFiles;
}
else{
return null;
}
}
private static MessageFile getMessageFile(String gumTreeInput, String fileName) {
// String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile
// File revFilesPath = new File(inputPath + "revFiles/");
// File[] revFiles = revFilesPath.listFiles(); // project folders
// List<MessageFile> msgFiles = new ArrayList<>();
// gumTreeInput = /Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/AMQP/
// fileName = 01534a_df5570_spring-rabbit#src#test#java#org#springframework#amqp#rabbit#listener#LocallyTransactedTests.java
// if (revFiles.length >= 0) {
// for (File revFile : revFiles) {
// if (revFile.getName().endsWith(".java")) {
// String fileName = revFile.getName();
File revFile = new File(gumTreeInput + "revFiles/"+fileName);
File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
fileName = fileName.replace(".java", ".txt");
File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file
// File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
return msgFile;
// msgFile.setPositionFile(positionFile);
// msgFiles.add(msgFile);
// }
// }
//
// return msgFiles;
// }
// else{
// return null;
// }
}
public static void classifyByAlarmTypes() {
final String alarmTypesFilePath = Configuration.ALARM_TYPES_FILE;
List<String> alarmTypes = readStringList(alarmTypesFilePath);
//edit scripts, sizes of edit scripts, buggy tokens, patches.
classifyByAlarmTypes(alarmTypes, Configuration.EDITSCRIPT_SIZES_FILE);
classifyByAlarmTypes(alarmTypes, Configuration.EDITSCRIPTS_FILE);
classifyByAlarmTypes(alarmTypes, Configuration.BUGGY_CODE_TOKENS_FILE);
classifyByAlarmTypes2(alarmTypes, Configuration.PATCH_SOURCECODE_FILE);
}
private static void classifyByAlarmTypes(List<String> alarmTypes, String file) {
Map<String, StringBuilder> buildersMap = new HashMap<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(file);
scanner = new Scanner(fis);
int counter = 0;
while (scanner.hasNextLine()) {
String alarmType = alarmTypes.get(counter);
StringBuilder builder = getBuilder(buildersMap, alarmType);
builder.append(scanner.nextLine() + "\n");
counter ++;
if (counter % 1000 == 0) {
outputBuilders(buildersMap, file);
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
outputBuilders(buildersMap, file);
}
private static void classifyByAlarmTypes2(List<String> alarmTypes, String patchSourcecodeFile) {
Map<String, StringBuilder> buildersMap = new HashMap<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(patchSourcecodeFile);
scanner = new Scanner(fis);
int counter = 0;
String singlePatch = "";
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
if (Configuration.PATCH_SIGNAL.equals(line)) {
if (!"".equals(singlePatch)) {
String alarmType = alarmTypes.get(counter);
StringBuilder builder = getBuilder(buildersMap, alarmType);
builder.append(scanner.nextLine() + "\n");
counter ++;
if (counter % 2000 == 0) {
outputBuilders(buildersMap, patchSourcecodeFile);
}
}
singlePatch = line + "\n";
}
singlePatch += line + "\n";
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
outputBuilders(buildersMap, patchSourcecodeFile);
}
private static void outputBuilders(Map<String, StringBuilder> map, String fileNameStr) {
File file = new File(fileNameStr);
String fileName = file.getName();
String parentPath = file.getParent();
for (Map.Entry<String, StringBuilder> entry : map.entrySet()) {
String alarmType = entry.getKey();
StringBuilder builder = entry.getValue();
FileHelper.outputToFile(parentPath + "/" + alarmType + "/" + fileName, builder, true);
builder.setLength(0);
entry.setValue(builder);
}
}
public static List<String> readStringList(String inputFile) {
List<String> list = new ArrayList<>();
FileInputStream fis = null;
Scanner scanner = null;
try {
fis = new FileInputStream(inputFile);
scanner = new Scanner(fis);
while(scanner.hasNextLine()) {
list.add(scanner.nextLine());
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} finally {
try {
scanner.close();
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return list;
}
private static StringBuilder getBuilder(Map<String, StringBuilder> buildersMap, String alarmType) {
if (buildersMap.containsKey(alarmType)) {
return buildersMap.get(alarmType);
} else {
StringBuilder builder = new StringBuilder();
buildersMap.put(alarmType, builder);
return builder;
}
}
}
src/main/java/edu/lu/uni/serval/Launcher.java
+60
View File
@@ -0,0 +1,60 @@
package edu.lu.uni.serval;
import edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader;
import edu.lu.uni.serval.FixPatternParser.cluster.CalculatePairs;
import edu.lu.uni.serval.FixPatternParser.cluster.ImportPairs2DB;
import edu.lu.uni.serval.FixPatternParser.cluster.StoreFile;
/**
* Created by anilkoyuncu on 14/04/2018.
*/
public class Launcher {
public static void main(String[] args) {
String inputPath;
String portInner;
String serverWait;
String dbDir;
String chunkName;
String numOfWorkers;
String jobType;
String port;
String outputPath;
if (args.length > 0) {
jobType = args[0];
inputPath = args[1];
portInner = args[2];
serverWait = args[3];
chunkName = args[4];
numOfWorkers = args[5];
dbDir = args[6];
port = args[6];
outputPath = args[7];
} else {
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs";
portInner = "6380";
serverWait = "10000";
chunkName = "dumps.rdb";
dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
numOfWorkers = "1";
jobType = "AKKA";
port = "6399";
outputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImport";
}
String parameters = String.format("\nJob %s \nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \nnumOfWorks %s \ndbDir %s", jobType, inputPath, portInner, serverWait, chunkName, numOfWorkers, dbDir);
switch (jobType){
case "STORE":
StoreFile.main(inputPath,portInner,serverWait,dbDir,chunkName,numOfWorkers);
case "CALCPAIRS":
CalculatePairs.main(inputPath,portInner,serverWait,dbDir,chunkName,numOfWorkers,port,outputPath);
case "IMPORTPAIRS":
ImportPairs2DB.main(inputPath,portInner,serverWait,dbDir,chunkName,numOfWorkers);
case "AKKA":
AkkaTreeLoader.main(inputPath,portInner,serverWait,dbDir,chunkName,numOfWorkers,port);
}
}
}
src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java
+3 -1
View File
@@ -203,10 +203,12 @@ public class HierarchicalRegrouper {
if (type != 83 && type != 77 && type != 78 && type != 79
&& type != 5 && type != 39 && type != 43 && type != 74 && type != 75
&& type != 76 && type != 84 && type != 87 && type != 88 && type != 42) {
// ArrayType, PrimitiveType, SimpleType, ParameterizedType,
// ArrayType, PrimitiveType, SimpleType, ParameterizedType,
// QualifiedType, WildcardType, UnionType, IntersectionType, NameQualifiedType, SimpleName
return null;
}
// } else if (parent.getType() == 31) { // method declaration
// int type = action.getNode().getType();
// if (Checker.isStatement(type)) {// statements