From c03061db694d08682a938b52bcb33b9940366901 Mon Sep 17 00:00:00 2001 From: Anil Koyuncu Date: Sat, 14 Apr 2018 19:07:53 +0200 Subject: [PATCH] launcher --- pom.xml | 2 +- .../cluster/AkkaTreeLoader.java | 82 ++-- .../cluster/CalculatePairs.java | 64 +-- .../cluster/ImportPairs2DB.java | 48 +-- .../FixPatternParser/cluster/StoreFile.java | 48 +-- .../MultiThreadTreeLoaderCluster3.java | 291 ++++++-------- .../violations/TestHunkParserSingleFile.java | 363 ++++++++++++++++++ src/main/java/edu/lu/uni/serval/Launcher.java | 60 +++ .../regroup/HierarchicalRegrouper.java | 4 +- 9 files changed, 677 insertions(+), 285 deletions(-) create mode 100644 src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParserSingleFile.java create mode 100644 src/main/java/edu/lu/uni/serval/Launcher.java diff --git a/pom.xml b/pom.xml index a091669..393feb7 100644 --- a/pom.xml +++ b/pom.xml @@ -108,7 +108,7 @@ - edu.lu.uni.serval.FixPatternParser.cluster.ImportPairs2DB + edu.lu.uni.serval.Launcher diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java index 8f6eee2..b27459f 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java @@ -92,48 +92,48 @@ public class AkkaTreeLoader { log.info("Load done"); } - public static void main(String[] args) { +// public static void main(String[] args) { + public static void main(String inputPath,String portInner,String serverWait,String dbDir,String chunkName,String numOfWorkers,String port){ - - String inputPath; -// String outputPath; - String port; - String portInner; -// String pairsCSVPath; - String importScript; -// String pairsCompletedPath; - String serverWait; -// String option; - String dbDir; - String chunkName; - String numOfWorkers; - if (args.length > 0) { - inputPath = args[0]; - portInner = args[1]; - serverWait = args[2]; -// option = args[4]; - chunkName = args[3]; - numOfWorkers = args[4]; - dbDir = args[5]; - port = args[6]; -// pairsCSVPath = args[3]; -// importScript = args[4]; -// pairsCompletedPath = args[3]; - } else { - inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; -// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/"; - port = "6399"; - portInner = "6380"; - serverWait = "10000"; -// option = "COMP"; -// pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test"; -// importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh"; -// pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed"; - chunkName ="chunk3.rdb"; - dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; - numOfWorkers = "1"; - - } +// String inputPath; +//// String outputPath; +// String port; +// String portInner; +//// String pairsCSVPath; +// String importScript; +//// String pairsCompletedPath; +// String serverWait; +//// String option; +// String dbDir; +// String chunkName; +// String numOfWorkers; +// if (args.length > 0) { +// inputPath = args[0]; +// portInner = args[1]; +// serverWait = args[2]; +//// option = args[4]; +// chunkName = args[3]; +// numOfWorkers = args[4]; +// dbDir = args[5]; +// port = args[6]; +//// pairsCSVPath = args[3]; +//// importScript = args[4]; +//// pairsCompletedPath = args[3]; +// } else { +// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; +//// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/"; +// port = "6399"; +// portInner = "6380"; +// serverWait = "10000"; +//// option = "COMP"; +//// pairsCSVPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test"; +//// importScript = "/Users/anilkoyuncu/bugStudy/dataset/pairs/test2.sh"; +//// pairsCompletedPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs_completed"; +// chunkName ="chunk3.rdb"; +// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; +// numOfWorkers = "1"; +// +// } String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \nnumOfWorks %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,numOfWorkers,dbDir); log.info(parameters); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java index 500445e..ccc4e95 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java @@ -29,39 +29,39 @@ import static edu.lu.uni.serval.FixPatternParser.cluster.TreeLoaderClusterL1.poo */ public class CalculatePairs { private static Logger log = LoggerFactory.getLogger(CalculatePairs.class); - public static void main(String[] args) { +// public static void main(String[] args) { + public static void main(String inputPath,String portInner,String serverWait,String dbDir,String chunkName,String numOfWorkers,String port,String outputPath){ - - String inputPath; - String port; - String portInner; - String serverWait; - String dbDir; - String chunkName; - String outputPath; - - if (args.length > 0) { - inputPath = args[0]; - portInner = args[1]; - serverWait = args[2]; - chunkName = args[3]; - - dbDir = args[5]; - port = args[6]; - outputPath = args[7]; - - } else { - inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; - - port = "6399"; - portInner = "6380"; - serverWait = "10000"; - outputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImport"; - chunkName ="chunk"; - dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; - - - } +// String inputPath; +// String port; +// String portInner; +// String serverWait; +// String dbDir; +// String chunkName; +// String outputPath; +// +// if (args.length > 0) { +// inputPath = args[0]; +// portInner = args[1]; +// serverWait = args[2]; +// chunkName = args[3]; +// +// dbDir = args[5]; +// port = args[6]; +// outputPath = args[7]; +// +// } else { +// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; +// +// port = "6399"; +// portInner = "6380"; +// serverWait = "10000"; +// outputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImport"; +// chunkName ="chunk"; +// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; +// +// +// } String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,dbDir); log.info(parameters); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java index 8922c67..a0fa800 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java @@ -19,31 +19,33 @@ import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedi */ public class ImportPairs2DB { private static Logger log = LoggerFactory.getLogger(ImportPairs2DB.class); - public static void main(String[] args) { +// public static void main(String[] args) { - String inputPath; - String portInner; - String serverWait; - String dbDir; - String chunkName; - String numOfWorkers; - if (args.length > 0) { - inputPath = args[0]; - portInner = args[1]; - serverWait = args[2]; - chunkName = args[3]; - numOfWorkers = args[4]; - dbDir = args[5]; - } else { - inputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs"; - portInner = "6380"; - serverWait = "10000"; - chunkName ="dumps.rdb"; - dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; - numOfWorkers = "1"; - } + public static void main(String inputPath,String portInner,String serverWait,String dbDir,String chunkName,String numOfWorkers){ + +// String inputPath; +// String portInner; +// String serverWait; +// String dbDir; +// String chunkName; +// String numOfWorkers; +// if (args.length > 0) { +// inputPath = args[0]; +// portInner = args[1]; +// serverWait = args[2]; +// chunkName = args[3]; +// numOfWorkers = args[4]; +// dbDir = args[5]; +// } else { +// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs"; +// portInner = "6380"; +// serverWait = "10000"; +// chunkName ="dumps.rdb"; +// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; +// numOfWorkers = "1"; +// } String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \nnumOfWorks %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,numOfWorkers,dbDir); - + log.info(parameters); File folder = new File(inputPath); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java index a8ff74a..c680d96 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java @@ -25,31 +25,31 @@ public class StoreFile { private static Logger log = LoggerFactory.getLogger(StoreFile.class); - public static void main(String[] args) { - - String inputPath; - String portInner; - String serverWait; - String dbDir; - String chunkName; - String numOfWorkers; - if (args.length > 0) { - inputPath = args[0]; - portInner = args[1]; - serverWait = args[2]; - chunkName = args[3]; - numOfWorkers = args[4]; - dbDir = args[5]; - } else { - inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; - portInner = "6399"; - serverWait = "10000"; - chunkName ="dumps.rdb"; - dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; - numOfWorkers = "1"; - } +// public static void main(String[] args) { + public static void main(String inputPath,String portInner,String serverWait,String dbDir,String chunkName,String numOfWorkers){ +// String inputPath; +// String portInner; +// String serverWait; +// String dbDir; +// String chunkName; +// String numOfWorkers; +// if (args.length > 0) { +// inputPath = args[0]; +// portInner = args[1]; +// serverWait = args[2]; +// chunkName = args[3]; +// numOfWorkers = args[4]; +// dbDir = args[5]; +// } else { +// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; +// portInner = "6399"; +// serverWait = "10000"; +// chunkName ="dumps.rdb"; +// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; +// numOfWorkers = "1"; +// } String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \nnumOfWorks %s \ndbDir %s",inputPath,portInner,serverWait,chunkName,numOfWorkers,dbDir); - + log.info(parameters); String cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; cmd = String.format(cmd, dbDir,chunkName,Integer.valueOf(portInner)); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java index ee301d2..65d7f04 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java @@ -12,6 +12,7 @@ import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper; import edu.lu.uni.serval.utils.FileHelper; import edu.lu.uni.serval.utils.ListSorter; +import org.apache.commons.lang3.StringUtils; import org.javatuples.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,7 +97,7 @@ public class MultiThreadTreeLoaderCluster3 { cmd = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; cmd = String.format(cmd, dbDir,"dumps.rdb",Integer.valueOf("6399")); - edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd,"10000"); + edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd,"1000"); mainCompare(inputPath,port,pairsCSVPath,importScript); // calculatePairs(inputPath, outputPath); @@ -176,8 +177,8 @@ public class MultiThreadTreeLoaderCluster3 { .collect(Collectors.toList()); for (File f:folders){ - - if(f.getName().startsWith("cluster7_0")) { + //36_0,119_0,4_0 +// if(f.getName().startsWith("cluster1_0")) { try (Jedis jedis = jedisPool.getResource()) { @@ -217,7 +218,7 @@ public class MultiThreadTreeLoaderCluster3 { - } +// } @@ -245,48 +246,9 @@ public class MultiThreadTreeLoaderCluster3 { } -// File folder = new File("/Users/anilkoyuncu/bugStudy/code/python/clusterDumps/"+cluster + "/" + s + ".txt_" + actionSetPosition); -// -// -// ITree tree = null; -// try { -// FileInputStream fi = new FileInputStream(folder); -// ObjectInputStream oi = new ObjectInputStream(fi); -// tree = (ITree) oi.readObject(); -// oi.close(); -// fi.close(); -// -// -// } catch (FileNotFoundException e) { -// log.error("File not found"); -// e.printStackTrace(); -// } catch (IOException e) { -// log.error("Error initializing stream"); -// e.printStackTrace(); -// } catch (ClassNotFoundException e) { -// // TODO Auto-generated catch block -// e.printStackTrace(); -// } -// return tree; - ITree tree = null; Jedis inner = null; -// String[] split2 = firstValue.split("/"); -// String cluster = split2[1]; -// -// String[] split1 = folders.get(0).getName().split(".txt_"); -// String s = split1[0]; -// String[] splitPJ = split1[1].split("_"); -// String project = splitPJ[1]; -// String actionSetPosition = splitPJ[0]; -// File folder = new File("/Users/anilkoyuncu/bugStudy/code/python/cluster/"+cluster); -// File folder = new File("/Users/anilkoyuncu/bugStudy/code/python/clusterDumps/"+cluster + "/" + s + ".txt_" + actionSetPosition); -// File[] listOfFiles = folder.listFiles(); -// Stream stream = Arrays.stream(listOfFiles); -// List folders = stream -// .filter(x -> !x.getName().startsWith(".") && x.getName().startsWith(split2[2])) -// .collect(Collectors.toList()); @@ -296,15 +258,6 @@ public class MultiThreadTreeLoaderCluster3 { String si= inner.get(fn); HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(si); -// ITree newTree = ((Update)actionSet.getAction()).getNewNode(); -// ITree oldTree = ((Update)actionSet.getAction()).getNode(); -// -// Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); -// m.match(); -// ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); -// ag.generate(); -// List actions = ag.getActions(); -// log.info(actions.toString()); ITree parent = null; ITree children =null; @@ -322,46 +275,10 @@ public class MultiThreadTreeLoaderCluster3 { inner.close(); } } -// Pair pair = new Pair<>(tree,project); + return tree; -// File[] listOfFiles = folder.listFiles(); -// Stream stream = Arrays.stream(listOfFiles); -// List folders = stream -// .filter(x -> !x.getName().startsWith(".") && x.getName().startsWith(split2[2])) -// .collect(Collectors.toList()); -// -// -// -//// String[] split1 = folders.get(0).getName().split(".txt_"); -//// String s = split1[0]; -//// String[] splitPJ = split1[1].split("_"); -//// String project = splitPJ[1]; -//// String actionSetPosition = splitPJ[0]; -// -// File prevFile = new File(gumTreeInput + project+ "/" + "prevFiles/prev_" + s + ".java");// previous file -// File revFile = new File(gumTreeInput + project+ "/" + "revFiles/" + s + ".java");//rev file -// -// List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); -// -// HierarchicalActionSet actionSet = actionSets.get(Integer.valueOf(actionSetPosition)); -//// for (HierarchicalActionSet actionSet : actionSets) { -// -// ITree test = getActionTree(actionSet); -// test.getLabel(); -// for (ITree descendant : test.getDescendants()) { -//// descendant.setLabel(""); -// if(!(descendant.getType() == 100 || descendant.getType() == 101 || descendant.getType() == 102 || descendant.getType() == 103)){ -// descendant.setType(104); -// } -// } -// test.getDescendants(); -// test.setParent(null); -//// } -//// } -// -// Pair pair = new Pair<>(test,project); -// return pair; + } public static ITree getASTTree(HierarchicalActionSet actionSet, ITree parent, ITree children,TreeContext tc){ @@ -436,22 +353,14 @@ public class MultiThreadTreeLoaderCluster3 { } private static List getNames(ITree oldTree, List oldTokens){ -// if((oldTree.getType() == 42 && oldTree.getLabel().startsWith("Name:")) || oldTree.getType() == 42 && oldTree.getParent().getType() == 59 || oldTree.getType() == 43 || (oldTree.getType() == 41 && oldTree.getLabel().startsWith("SimpleName:")) ){ -// -// oldTokens.add(oldTree.getLabel()); -// -// } -// for (ITree oldDescendant : oldTree.getDescendants()) { -// if ((oldDescendant.getType() == 42 && oldDescendant.getLabel().startsWith("Name:") ) || oldDescendant.getType() == 42 && oldDescendant.getParent().getType() == 59 ||oldDescendant.getType() == 43 || (oldDescendant.getType() == 41 && oldDescendant.getLabel().startsWith("SimpleName:"))){ -// -// oldTokens.add(oldDescendant.getLabel()); -// -// } -// } + List descendants = oldTree.getDescendants(); descendants.add(0,oldTree); boolean upd=false; + for (ITree oldDescendant : descendants) { + + boolean addToken = false; int type = oldDescendant.getType(); String sType = String.valueOf(type); @@ -460,18 +369,18 @@ public class MultiThreadTreeLoaderCluster3 { (sType.equals("32") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1) || (sType.equals("59") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1) || (sType.equals("43") && oldDescendant.getHeight() == 0 && oldDescendant.getChildren().size() == 0) || - (sType.equals("14") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1) || + (sType.equals("53") )|| //&& oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1) || (sType.equals("7") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1) || - (sType.equals("27") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1) + (sType.equals("41") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1) ){ - int depth = oldDescendant.getChildren().size(); +// int depth = oldDescendant.getChildren().size(); String label = oldDescendant.getLabel(); - if(sType.equals("32") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1){ - log.info(""); - } +// if(sType.equals("32") && oldDescendant.getHeight() == 1 && oldDescendant.getChildren().size() == 1){ +// log.info(""); +// } label = label.split("@AT@")[0]; @@ -480,15 +389,50 @@ public class MultiThreadTreeLoaderCluster3 { List m = new ArrayList(); if(label.startsWith("UPD")){ upd = true; - String timeRegex = "@@(.*)@TO@(.*)"; - Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL); - java.util.regex.Matcher matcher = pattern.matcher(split[1]); - + java.util.regex.Matcher matcher; + if(sType.equals("53")){ + String timeRegex = ".*@@(ClassInstanceCreation:new [a-zA-Z0-9]+).*@TO@\\s(ClassInstanceCreation:new [a-zA-Z0-9]+).*"; + Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL); + matcher= pattern.matcher(split[1]); + }else { + String timeRegex = "@@(.*)@TO@(.*)"; + Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL); + matcher = pattern.matcher(split[1]); + } if (matcher.matches()) { String hours = matcher.group(1); String to = matcher.group(2); - m.add(hours.trim()); - m.add(to.trim()); + if(sType.equals("31")){ + + String commonPrefix = StringUtils.getCommonPrefix(hours.trim(), to.trim()); + if(commonPrefix.isEmpty()){ + log.info("PREFIX EMPTY"); + }else { + String s = hours.trim().replace(commonPrefix, ""); + String s1 = to.trim().replace(commonPrefix, ""); + String[] split1 = s.split(","); + String[] split3 = s1.split(","); + Set set = new TreeSet<>(); + for (String s2 : split1) { + if(!s2.isEmpty()) { + set.add(s2.trim()); + } + } + for (String s2 : split3) { + if(!s2.isEmpty()) { + set.add(s2.trim()); + } + } + + List list = set.stream().collect(Collectors.toList()); + m.addAll(list); + addToken = true; + + } + }else{ + m.add(hours.trim()); + m.add(to.trim()); + } } }else if(label.startsWith("INS") && upd == false){ @@ -521,25 +465,67 @@ public class MultiThreadTreeLoaderCluster3 { } } } - + boolean alreadyAddParentMethodName = false; for (String s : m) { // if(s.isEmpty()){ // continue; // } //TODO remove 44 if(s.startsWith("SimpleName:") || s.startsWith("Name:")){ - oldTokens.add(s); - }else if (s.startsWith("MethodName:")){ String methodName = s.split(":")[1]; oldTokens.add(methodName); - }else if( sType.equals("59") || sType.equals("43")|| sType.equals("14") || sType.equals("7") || sType.equals("27") || sType.equals("83") || sType.equals("44")){ - if(sType.equals("27") || sType.equals("83") || sType.equals("44")){ - oldTokens.add(s); - } - else { - String s1 = s.split("=")[0]; - oldTokens.add(s1); + }else if (s.startsWith("MethodName:")){ + String methodName = s.split(":")[1]; +// ITree parent = oldDescendant.getParent(); +// +// if(parent!= null && parent.getType() == 32 && !alreadyAddParentMethodName){ //parent is method invocation statement +// String parentLabel = parent.getLabel(); +// String[] pns = parentLabel.split("\\." + methodName); +// if(pns.length > 1) { +// String parentName = pns[0]; +// String[] parentNameSplit = parentName.split("@@"); +// if (parentNameSplit.length > 1) { +// String parentMethodName = parentNameSplit[1]; +// String s1 = parentMethodName.split("@TO@")[0]; +// oldTokens.add(s1.trim()); +// alreadyAddParentMethodName = true; +// } +// } +// +// } + oldTokens.add(methodName); + }else if( sType.equals("59") || sType.equals("43")|| sType.equals("53") || sType.equals("7") || sType.equals("27") || sType.equals("83") || sType.equals("44") ||sType.equals("78") || sType.equals("41") || addToken){ +// if(sType.equals("27") || sType.equals("83") || sType.equals("44")){ +// String parentLabel = oldDescendant.getParent().getLabel(); +// List parentM = new ArrayList(); +// if(parentLabel.startsWith("UPD")) { +// upd = true; +// String timeRegex = "@@(.*)@TO@(.*)"; +// Pattern pattern = Pattern.compile(timeRegex, Pattern.DOTALL); +// java.util.regex.Matcher matcher = pattern.matcher(split[1]); +// +// if (matcher.matches()) { +// String hours = matcher.group(1); +// String to = matcher.group(2); +// parentM.add(hours.trim()); +// parentM.add(to.trim()); +// +// } +// } +// +// +// oldTokens.add(s); +// }else + if(sType.equals("53") || sType.equals("78") || addToken){//sType.equals("41") || + + oldTokens.add(s); + + } +// else { +// String s1 = s.split("=")[0]; +// oldTokens.add(s1); +// } } // else // if(oldTokens.size() < 2){ @@ -550,9 +536,9 @@ public class MultiThreadTreeLoaderCluster3 { } } - if (oldTokens.size() == 0){ - log.info("dur bakalim nereye!???"); - } +// if (oldTokens.size() == 0 || oldTokens.size() > 3) {// && (oldTree.getType() != 41 && oldTree.getType() != 21 && oldTree.getType() !=17 && oldTree.getType()!=60 && oldTree.getType() != 46)){ +// log.info("dur bakalim nereye!???"); +// } return oldTokens; } @@ -580,9 +566,9 @@ public class MultiThreadTreeLoaderCluster3 { String firstValue = resultMap.get("0"); String secondValue = resultMap.get("1"); - if(firstValue.equals("7/0/7af9e0_e674f1_spring-social-web#src#main#java#org#springframework#social#connect#web#ConnectController.txt_0_SOCIAL") || secondValue.equals("7/0/7af9e0_e674f1_spring-social-web#src#main#java#org#springframework#social#connect#web#ConnectController.txt_0_SOCIAL")){ - log.info(""); - } +// if(firstValue.equals("7/0/7af9e0_e674f1_spring-social-web#src#main#java#org#springframework#social#connect#web#ConnectController.txt_0_SOCIAL") || secondValue.equals("7/0/7af9e0_e674f1_spring-social-web#src#main#java#org#springframework#social#connect#web#ConnectController.txt_0_SOCIAL")){ +// log.info(""); +// } ///35/1/22b5f7_84bf27_ui#org.eclipse.pde.runtime#src#org#eclipse#pde#internal#runtime#registry#RegistryBrowserLabelProvider.txt_2_PDE @@ -635,6 +621,10 @@ public class MultiThreadTreeLoaderCluster3 { oldTokens = getNames(oldTree,oldTokens); newTokens = getNames(newTree,newTokens); + + if(oldTokens.size() == 0 || newTokens.size() == 0){ + log.error("Cluster {} has no tokens on pair {}",clusterName , name); + } // Matcher m = Matchers.getInstance().getMatcher(oldTree, newTree); // m.match(); CharSequence[] oldSequences = oldTokens.toArray(new CharSequence[oldTokens.size()]); @@ -648,17 +638,21 @@ public class MultiThreadTreeLoaderCluster3 { } }else{ overallSimi = Double.valueOf(0); - if(oldSequences.length != 0) { - log.info("ERROR"); - } +// if(oldSequences.length != 0) { +// log.info("ERROR"); +// } } - if(overallSimi.equals(1.0)){ + int retval = Double.compare(overallSimi, Double.valueOf(0.8)); + if(retval >= 0){ // log.info("YES"); // log.info(name); // log.info(firstValue); // log.info(secondValue); // log.info("************"); + if(!overallSimi.equals(1.0)){ + log.info(""); + } String matchKey = "match-"+clusterName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); String result = firstValue + "," + secondValue + ","+String.join(",", oldTokens); jedis.select(1); @@ -670,35 +664,6 @@ public class MultiThreadTreeLoaderCluster3 { - -// ActionGenerator ag = new ActionGenerator(oldTree, newTree, m.getMappings()); -// ag.generate(); -// List actions = ag.getActions(); -// -// String resultKey = "result_" + (String.valueOf(i)) + "_" + String.valueOf(j); -// double chawatheSimilarity1 = m.chawatheSimilarity(oldTree, newTree); -// String chawatheSimilarity = String.format("%1.2f", chawatheSimilarity1); -// double diceSimilarity1 = m.diceSimilarity(oldTree, newTree); -// String diceSimilarity = String.format("%1.2f", diceSimilarity1); -// double jaccardSimilarity1 = m.jaccardSimilarity(oldTree, newTree); -// String jaccardSimilarity = String.format("%1.2f", jaccardSimilarity1); -// -// String editDistance = String.valueOf(actions.size()); -// jedis.select(1); -// String result = resultMap.get("0") + "," + oldProject +"," + resultMap.get("1") + "," +newProject+ "," + chawatheSimilarity + "," + diceSimilarity + "," + jaccardSimilarity + "," + editDistance; -//// jedis.set(resultKey, result); -// -// if (((Double) chawatheSimilarity1).equals(1.0) || ((Double) diceSimilarity1).equals(1.0) -// || ((Double) jaccardSimilarity1).equals(1.0) || actions.size() == 0) { -// String matchKey = "match-"+clusterName+"_" + (String.valueOf(i)) + "_" + String.valueOf(j); -// jedis.select(1); -// jedis.set(matchKey, result); -// } -// jedis.select(0); - -// -//// log.info("Completed " + resultKey); - }catch (Exception e){ log.error(e.toString() + " {}",(name)); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParserSingleFile.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParserSingleFile.java new file mode 100644 index 0000000..0266e2a --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParserSingleFile.java @@ -0,0 +1,363 @@ +package edu.lu.uni.serval.FixPatternParser.violations; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; +import edu.lu.uni.serval.MultipleThreadsParser.ParseFixPatternActor; +import edu.lu.uni.serval.MultipleThreadsParser.WorkMessage; +import edu.lu.uni.serval.config.Configuration; +import edu.lu.uni.serval.utils.FileHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class TestHunkParserSingleFile { + + private static Logger log = LoggerFactory.getLogger(TestHunkParserSingleFile.class); + public static void main(String[] args) { + // input data + +// String rootPath = "/Users/anilkoyuncu/bugStudy"; + String inputPath; + String outputPath; + if(args.length > 0){ + inputPath = args[1]; + outputPath = args[0]; + }else{ +// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeInputBug4"; + inputPath = "/Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4"; +// outputPath = "/Users/anilkoyuncu/bugStudy/code/python/GumTreeOutput2/"; + outputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputSingle"; + } + + //5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.txt_1_CAMEL + + File folder = new File(inputPath); + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List folders = stream + .filter(x -> !x.getName().startsWith(".")) + + .collect(Collectors.toList()); + +// List targetList = new ArrayList(); +// for (File f:folders){ +// for(File f1 :f.listFiles()){ +// if (!f1.getName().startsWith(".")){ +// targetList.add(f1); +// } +// } +// } + +// List pjList = Arrays.asList("DATAJPA","ZXing","PDE","SWS","SWT", "SWF", "COLLECTIONS", "JDT"); + List files = new ArrayList(); +// files.add("5d9d60_76f5be_components#camel-jaxb#src#test#java#org#apache#camel#jaxb#FallbackTypeConverterShouldNotThrowExceptionTest.java"); + files.add("49821e_3cc787_components#camel-infinispan#src#main#java#org#apache#camel#component#infinispan#InfinispanConstants.java"); + for(String f : files){ + String pjName = "CAMEL"; +// for (File target : folders) { +// String pjName = target.getName(); +// if (!pjList.contains(pjName)){ +// continue; +// } + +// final List msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/" + MessageFile messageFile = getMessageFile(inputPath + "/" + pjName +"/", f); + + List msgFiles = new ArrayList<>(); + msgFiles.add(messageFile); + String GUM_TREE_OUTPUT = outputPath + "/"+ pjName + "/"; + final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list"; + final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list"; + final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list"; + final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv"; + final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list"; + + + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ActionSetDumps"); + FileHelper.deleteDirectory(editScriptsFilePath); + FileHelper.deleteDirectory(patchesSourceCodeFilePath); + FileHelper.deleteDirectory(buggyTokensFilePath); + FileHelper.deleteDirectory(editScriptSizesFilePath); + FileHelper.deleteDirectory(alarmTypesFilePath); + + StringBuilder astEditScripts = new StringBuilder(); + StringBuilder tokens = new StringBuilder(); + StringBuilder sizes = new StringBuilder(); + StringBuilder patches = new StringBuilder(); + StringBuilder alarmTypes = new StringBuilder(); + +// int a = 0; + + ActorSystem system = null; + ActorRef parsingActor = null; + final WorkMessage msg = new WorkMessage(0, msgFiles); + try { + log.info("Akka begins..."); + system = ActorSystem.create("Mining-FixPattern-System"); + parsingActor = system.actorOf(ParseFixPatternActor.props(1, editScriptsFilePath, + patchesSourceCodeFilePath, buggyTokensFilePath, editScriptSizesFilePath), "mine-fix-pattern-actor"); + parsingActor.tell(msg, ActorRef.noSender()); + } catch (Exception e) { + system.shutdown(); + e.printStackTrace(); + } + +// int counter = 0; +// for (MessageFile msgFile : msgFiles) { +// FixedViolationHunkParser parser = new FixedViolationHunkParser(); +// +// final ExecutorService executor = Executors.newSingleThreadExecutor(); +// // schedule the work +// final Future future = executor.submit(new RunnableParser(msgFile.getPrevFile(), +// msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser)); +// try { +// // where we wait for task to complete +// future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); +// String editScripts = parser.getAstEditScripts(); +// if (!editScripts.equals("")) { +// astEditScripts.append(editScripts); +// tokens.append(parser.getTokensOfSourceCode()); +// sizes.append(parser.getSizes()); +// patches.append(parser.getPatchesSourceCode()); +// alarmTypes.append(parser.getAlarmTypes()); +// +// a++; +// if (a % 100 == 0) { +// FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); +// FileHelper.outputToFile(buggyTokensFilePath, tokens, true); +// FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); +// FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); +// FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); +// astEditScripts.setLength(0); +// tokens.setLength(0); +// sizes.setLength(0); +// patches.setLength(0); +// alarmTypes.setLength(0); +// System.out.println("Finish of parsing " + a + " files......"); +// } +// } +// } catch (TimeoutException e) { +// err.println("task timed out"); +// future.cancel(true /* mayInterruptIfRunning */); +// } catch (InterruptedException e) { +// err.println("task interrupted"); +// } catch (ExecutionException e) { +// err.println("task aborted"); +// } finally { +// executor.shutdownNow(); +// } +// } + + FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); + FileHelper.outputToFile(buggyTokensFilePath, tokens, true); + FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); + FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); + FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); + astEditScripts.setLength(0); + tokens.setLength(0); + sizes.setLength(0); + patches.setLength(0); + alarmTypes.setLength(0); + + +// classifyByAlarmTypes(); + } + } + + + private static List getMessageFiles(String gumTreeInput) { + String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile + File revFilesPath = new File(inputPath + "revFiles/"); + File[] revFiles = revFilesPath.listFiles(); // project folders + List msgFiles = new ArrayList<>(); + + // gumTreeInput = /Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/AMQP/ + // fileName = 01534a_df5570_spring-rabbit#src#test#java#org#springframework#amqp#rabbit#listener#LocallyTransactedTests.java + if (revFiles.length >= 0) { + for (File revFile : revFiles) { +// if (revFile.getName().endsWith(".java")) { + String fileName = revFile.getName(); + File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file + fileName = fileName.replace(".java", ".txt"); + File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file + File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file + MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); + msgFile.setPositionFile(positionFile); + msgFiles.add(msgFile); +// } + } + + return msgFiles; + } + else{ + return null; + } + } + + private static MessageFile getMessageFile(String gumTreeInput, String fileName) { +// String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile +// File revFilesPath = new File(inputPath + "revFiles/"); +// File[] revFiles = revFilesPath.listFiles(); // project folders +// List msgFiles = new ArrayList<>(); + + // gumTreeInput = /Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/AMQP/ + // fileName = 01534a_df5570_spring-rabbit#src#test#java#org#springframework#amqp#rabbit#listener#LocallyTransactedTests.java +// if (revFiles.length >= 0) { +// for (File revFile : revFiles) { +// if (revFile.getName().endsWith(".java")) { +// String fileName = revFile.getName(); + File revFile = new File(gumTreeInput + "revFiles/"+fileName); + File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file + fileName = fileName.replace(".java", ".txt"); + File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file +// File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file + MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); + return msgFile; +// msgFile.setPositionFile(positionFile); +// msgFiles.add(msgFile); +// } +// } +// +// return msgFiles; +// } +// else{ +// return null; +// } + } + + public static void classifyByAlarmTypes() { + + final String alarmTypesFilePath = Configuration.ALARM_TYPES_FILE; + List alarmTypes = readStringList(alarmTypesFilePath); + //edit scripts, sizes of edit scripts, buggy tokens, patches. + classifyByAlarmTypes(alarmTypes, Configuration.EDITSCRIPT_SIZES_FILE); + classifyByAlarmTypes(alarmTypes, Configuration.EDITSCRIPTS_FILE); + classifyByAlarmTypes(alarmTypes, Configuration.BUGGY_CODE_TOKENS_FILE); + classifyByAlarmTypes2(alarmTypes, Configuration.PATCH_SOURCECODE_FILE); + } + + private static void classifyByAlarmTypes(List alarmTypes, String file) { + Map buildersMap = new HashMap<>(); + FileInputStream fis = null; + Scanner scanner = null; + try { + fis = new FileInputStream(file); + scanner = new Scanner(fis); + int counter = 0; + while (scanner.hasNextLine()) { + String alarmType = alarmTypes.get(counter); + StringBuilder builder = getBuilder(buildersMap, alarmType); + builder.append(scanner.nextLine() + "\n"); + counter ++; + if (counter % 1000 == 0) { + outputBuilders(buildersMap, file); + } + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + try { + scanner.close(); + fis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + outputBuilders(buildersMap, file); + } + + private static void classifyByAlarmTypes2(List alarmTypes, String patchSourcecodeFile) { + Map buildersMap = new HashMap<>(); + FileInputStream fis = null; + Scanner scanner = null; + try { + fis = new FileInputStream(patchSourcecodeFile); + scanner = new Scanner(fis); + int counter = 0; + String singlePatch = ""; + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + if (Configuration.PATCH_SIGNAL.equals(line)) { + if (!"".equals(singlePatch)) { + String alarmType = alarmTypes.get(counter); + StringBuilder builder = getBuilder(buildersMap, alarmType); + builder.append(scanner.nextLine() + "\n"); + counter ++; + if (counter % 2000 == 0) { + outputBuilders(buildersMap, patchSourcecodeFile); + } + } + singlePatch = line + "\n"; + } + singlePatch += line + "\n"; + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + try { + scanner.close(); + fis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + outputBuilders(buildersMap, patchSourcecodeFile); + } + + private static void outputBuilders(Map map, String fileNameStr) { + File file = new File(fileNameStr); + String fileName = file.getName(); + String parentPath = file.getParent(); + for (Map.Entry entry : map.entrySet()) { + String alarmType = entry.getKey(); + StringBuilder builder = entry.getValue(); + + FileHelper.outputToFile(parentPath + "/" + alarmType + "/" + fileName, builder, true); + + builder.setLength(0); + entry.setValue(builder); + } + } + + public static List readStringList(String inputFile) { + List list = new ArrayList<>(); + FileInputStream fis = null; + Scanner scanner = null; + try { + fis = new FileInputStream(inputFile); + scanner = new Scanner(fis); + while(scanner.hasNextLine()) { + list.add(scanner.nextLine()); + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + try { + scanner.close(); + fis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return list; + } + + private static StringBuilder getBuilder(Map buildersMap, String alarmType) { + if (buildersMap.containsKey(alarmType)) { + return buildersMap.get(alarmType); + } else { + StringBuilder builder = new StringBuilder(); + buildersMap.put(alarmType, builder); + return builder; + } + } +} diff --git a/src/main/java/edu/lu/uni/serval/Launcher.java b/src/main/java/edu/lu/uni/serval/Launcher.java new file mode 100644 index 0000000..f5bf52e --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/Launcher.java @@ -0,0 +1,60 @@ +package edu.lu.uni.serval; + +import edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader; +import edu.lu.uni.serval.FixPatternParser.cluster.CalculatePairs; +import edu.lu.uni.serval.FixPatternParser.cluster.ImportPairs2DB; +import edu.lu.uni.serval.FixPatternParser.cluster.StoreFile; + +/** + * Created by anilkoyuncu on 14/04/2018. + */ +public class Launcher { + + public static void main(String[] args) { + + String inputPath; + String portInner; + String serverWait; + String dbDir; + String chunkName; + String numOfWorkers; + String jobType; + String port; + String outputPath; + if (args.length > 0) { + jobType = args[0]; + inputPath = args[1]; + portInner = args[2]; + serverWait = args[3]; + chunkName = args[4]; + numOfWorkers = args[5]; + dbDir = args[6]; + port = args[6]; + outputPath = args[7]; + } else { + inputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs"; + portInner = "6380"; + serverWait = "10000"; + chunkName = "dumps.rdb"; + dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; + numOfWorkers = "1"; + jobType = "AKKA"; + port = "6399"; + outputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImport"; + } + String parameters = String.format("\nJob %s \nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \nnumOfWorks %s \ndbDir %s", jobType, inputPath, portInner, serverWait, chunkName, numOfWorkers, dbDir); + + + switch (jobType){ + case "STORE": + StoreFile.main(inputPath,portInner,serverWait,dbDir,chunkName,numOfWorkers); + case "CALCPAIRS": + CalculatePairs.main(inputPath,portInner,serverWait,dbDir,chunkName,numOfWorkers,port,outputPath); + case "IMPORTPAIRS": + ImportPairs2DB.main(inputPath,portInner,serverWait,dbDir,chunkName,numOfWorkers); + case "AKKA": + AkkaTreeLoader.main(inputPath,portInner,serverWait,dbDir,chunkName,numOfWorkers,port); + } + } + +} diff --git a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java b/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java index 6ccb32e..c00f09e 100644 --- a/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java +++ b/src/main/java/edu/lu/uni/serval/gumtree/regroup/HierarchicalRegrouper.java @@ -203,10 +203,12 @@ public class HierarchicalRegrouper { if (type != 83 && type != 77 && type != 78 && type != 79 && type != 5 && type != 39 && type != 43 && type != 74 && type != 75 && type != 76 && type != 84 && type != 87 && type != 88 && type != 42) { - // ArrayType, PrimitiveType, SimpleType, ParameterizedType, + // ArrayType, PrimitiveType, SimpleType, ParameterizedType, // QualifiedType, WildcardType, UnionType, IntersectionType, NameQualifiedType, SimpleName return null; } + + // } else if (parent.getType() == 31) { // method declaration // int type = action.getNode().getType(); // if (Checker.isStatement(type)) {// statements