From 30f33270c158df5dc45a90ab5e254edcf27665c2 Mon Sep 17 00:00:00 2001 From: Anil Koyuncu Date: Wed, 18 Apr 2018 18:54:49 +0200 Subject: [PATCH] new dataset --- .../cluster/TreeLoaderClusterL1.java | 14 +-- .../violations/FixedViolationHunkParser.java | 2 + .../MultiThreadTreeLoaderCluster.java | 28 +++--- .../MultiThreadTreeLoaderCluster3.java | 92 +++++++++++++------ src/main/java/edu/lu/uni/serval/Launcher.java | 19 ++-- 5 files changed, 100 insertions(+), 55 deletions(-) diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java index 431dc84..c81d8f2 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/TreeLoaderClusterL1.java @@ -21,13 +21,9 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.time.Duration; import java.util.*; -import java.util.concurrent.Executors; -import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; -import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; - /** * Created by anilkoyuncu on 19/03/2018. */ @@ -39,7 +35,7 @@ public class TreeLoaderClusterL1 { private static Logger log = LoggerFactory.getLogger(TreeLoaderClusterL1.class); // public static void main(String[] args){ - public static void main(String portInner,String serverWait,String port,String inputPath,String level1DB,String level1Path) throws Exception { + public static void main(String portInner,String serverWait,String port,String inputPath,String level1DB,String level1Path,String innerTypePrefix) throws Exception { // String inputPath; // String outputPath; @@ -81,6 +77,7 @@ public class TreeLoaderClusterL1 { Stream stream = Arrays.stream(listFiles); List dbs = stream .filter(x -> x.getName().endsWith(".rdb")) + .filter(x-> x.getName().startsWith(innerTypePrefix)) .collect(Collectors.toList()); for (File db : dbs) { String cmdInner = "bash "+inputPath + "/" + "startServer.sh" +" %s %s %s"; @@ -137,12 +134,17 @@ public class TreeLoaderClusterL1 { innerPool.close(); } - String stopServer = "bash "+level1Path + "/" + "stopServer.sh" +" %s"; + String stopServer = "bash "+inputPath + "/" + "stopServer.sh" +" %s"; stopServer = String.format(stopServer,Integer.valueOf(portInner)); // loadRedis(stopServer,serverWait); cs.runShell(stopServer,serverWait); } + String stopServer1 = "bash "+inputPath + "/" + "stopServer.sh" +" %s"; + stopServer1 = String.format(stopServer1,Integer.valueOf(port)); +// loadRedis(stopServer,serverWait); + cs.runShell(stopServer1,serverWait); + // calculatePairsOfClusters(inputPath, outputPath); // mainCompare(inputPath,port,pairsCSVPath,importScript); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java index 0f51308..4d6e156 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/FixedViolationHunkParser.java @@ -95,6 +95,8 @@ public class FixedViolationHunkParser extends FixedViolationParser { } } + + // public void parseFixPatterns(File prevFile, File revFile, File diffentryFile) { // // GumTree results // List actionSets = parseChangedSourceCodeWithGumTree2(prevFile, revFile); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster.java index a7734f3..2c788b7 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster.java @@ -119,15 +119,18 @@ public class MultiThreadTreeLoaderCluster { log.info("Load done"); } - public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner) { + public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String serverWait,String type) throws Exception { + CallShell cs = new CallShell(); String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner)); - edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000"); +// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000"); + cs.runShell(cmd1,serverWait); -// String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; -// cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port)); + String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; + cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port)); + cs.runShell(cmd2,serverWait); // edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000"); @@ -164,7 +167,8 @@ public class MultiThreadTreeLoaderCluster { if (size == 0) { String comd = String.format(cmd3,f.getPath(),portInner); - loadRedis(comd); +// loadRedis(comd); + cs.runShell(comd); scan = jedis.scan("0", sc); size = scan.getResult().size(); @@ -179,7 +183,7 @@ public class MultiThreadTreeLoaderCluster { //76 scan.getResult().parallelStream() - .forEach(m -> coreCompare(m, jedisPool, clusterName,outerPool)); + .forEach(m -> coreCompare(m, jedisPool, clusterName,outerPool,type)); jedis.save(); @@ -207,7 +211,7 @@ public class MultiThreadTreeLoaderCluster { } - public static Pair getTree(String firstValue, JedisPool outerPool){ + public static Pair getTree(String firstValue, JedisPool outerPool,String type){ // HierarchicalActionSet actionSet = null; @@ -245,7 +249,7 @@ public class MultiThreadTreeLoaderCluster { try { inner = outerPool.getResource(); - String filename = project + "/ActionSetDumps/" + pureFileName + ".txt_" + actionSetPosition; + String filename = project + "/"+type+"/" + pureFileName + ".txt_" + actionSetPosition; String si= inner.get(filename); HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(si); @@ -405,7 +409,7 @@ public class MultiThreadTreeLoaderCluster { - private static void coreCompare(String name , JedisPool jedisPool,String clusterName,JedisPool outerPool) { + private static void coreCompare(String name , JedisPool jedisPool,String clusterName,JedisPool outerPool,String type) { try (Jedis jedis = jedisPool.getResource()) { @@ -459,8 +463,8 @@ public class MultiThreadTreeLoaderCluster { // } try { - Pair oldPair = getTree(firstValue, outerPool); - Pair newPair = getTree(secondValue, outerPool); + Pair oldPair = getTree(firstValue, outerPool,type); + Pair newPair = getTree(secondValue, outerPool,type); ITree oldTree = oldPair.getValue0(); ITree newTree = newPair.getValue0(); @@ -631,7 +635,7 @@ orginal calculate pairs, from all dumps of the projects String line = null; try { - FileOutputStream fos = new FileOutputStream(outputPath + "pairs/" +filename+".txt"); + FileOutputStream fos = new FileOutputStream(outputPath + "/pairs/" +filename+".txt"); DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java index 17c1639..2aa3097 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java @@ -161,17 +161,18 @@ public class MultiThreadTreeLoaderCluster3 { } // public static void mainCompare(String inputPath,String port,String pairsCSVPath,String importScript) { - public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner) { + public static void mainCompare(String port,String pairsCSVPath,String importScript,String dbDir,String chunkName,String dumpName,String portInner,String serverWait, String type) throws Exception { + CallShell cs = new CallShell(); String cmd1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; cmd1 = String.format(cmd1, dbDir,chunkName,Integer.valueOf(portInner)); - edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000"); - +// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd1,"1000"); + cs.runShell(cmd1,serverWait); String cmd2 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; cmd2 = String.format(cmd2, dbDir,dumpName,Integer.valueOf(port)); - edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000"); - +// edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis(cmd2,"10000"); + cs.runShell(cmd2,serverWait); String cmd3; cmd3 = "bash " + importScript +" %s %s"; @@ -207,8 +208,9 @@ public class MultiThreadTreeLoaderCluster3 { if (size == 0) { String comd = String.format(cmd3, f.getPath(), portInner); - edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster. - loadRedis(comd); + cs.runShell(comd); +// edu.lu.uni.serval.FixPatternParser.violations.MultiThreadTreeLoaderCluster. +// loadRedis(comd); scan = jedis.scan("0", sc); size = scan.getResult().size(); @@ -221,7 +223,7 @@ public class MultiThreadTreeLoaderCluster3 { scan.getResult().parallelStream() - .forEach(m -> coreCompare(m, jedisPool, clusterName, outerPool)); + .forEach(m -> coreCompare(m, jedisPool, clusterName, outerPool,type)); jedis.save(); @@ -242,7 +244,7 @@ public class MultiThreadTreeLoaderCluster3 { - public static ITree getTree(String firstValue, JedisPool outerPool){ + public static ITree getTree(String firstValue, JedisPool outerPool,String type){ // String gumTreeInput = "/Volumes/data/bugStudy_backup/dataset/GumTreeInputBug4/"; String[] split2 = firstValue.split("/"); String cluster = split2[1]; @@ -268,7 +270,7 @@ public class MultiThreadTreeLoaderCluster3 { try { inner = outerPool.getResource(); - String fn = project + "/ActionSetDumps/" + s + ".txt_" + actionSetPosition; + String fn = project + "/"+type+"/" + s + ".txt_" + actionSetPosition; String si= inner.get(fn); HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(si); @@ -443,6 +445,36 @@ public class MultiThreadTreeLoaderCluster3 { addToken = true; } + }else if(sType.equals("7")){//assignment + m.add(hours.trim().split("=")[0]); + m.add(to.trim().split("=")[0]); + addToken = true; + + }else if(sType.equals("43") && oldDescendant.getParent() != null && oldDescendant.getParent().getParent() != null){// catch clause + if(oldDescendant.getParent().getType() == 44 && oldDescendant.getParent().getParent().getType() == 12){ + m.add(hours.trim()); + m.add(to.trim()); + addToken = true; + } + }else if(sType.equals("59") && oldDescendant.getChildren().size()==1 && oldDescendant.getHeight()==1 ){ + if( oldDescendant.getChild(0).getType() == 34 || oldDescendant.getChild(0).getType() == 45) { + m.add(hours.trim().split("=")[0]); + m.add(to.trim().split("=")[0]); + addToken = true; + } + }else if(sType.equals("42") && oldDescendant.getParent() != null && oldDescendant.getParent().getType() == 27 ){//infix varuable change + + m.add(hours.trim()); + m.add(to.trim()); + addToken = true; + }else if(sType.equals("-1") && oldDescendant.getChildren().size()==0 && oldDescendant.getHeight()==0 ){//operator + m.add(hours.trim()); + m.add(to.trim()); + addToken = true; + }else if(sType.equals("83") && oldDescendant.getChildren().size()==0 && oldDescendant.getHeight()==0 ){//modifier + m.add(hours.trim()); + m.add(to.trim()); + addToken = true; }else{ m.add(hours.trim()); m.add(to.trim()); @@ -504,23 +536,23 @@ public class MultiThreadTreeLoaderCluster3 { oldTokens.add(methodName); }else if (s.startsWith("MethodName:")){ String methodName = s.split(":")[1]; -// ITree parent = oldDescendant.getParent(); -// -// if(parent!= null && parent.getType() == 32 && !alreadyAddParentMethodName){ //parent is method invocation statement -// String parentLabel = parent.getLabel(); -// String[] pns = parentLabel.split("\\." + methodName); -// if(pns.length > 1) { -// String parentName = pns[0]; -// String[] parentNameSplit = parentName.split("@@"); -// if (parentNameSplit.length > 1) { -// String parentMethodName = parentNameSplit[1]; -// String s1 = parentMethodName.split("@TO@")[0]; -// oldTokens.add(s1.trim()); -// alreadyAddParentMethodName = true; -// } -// } -// -// } + ITree parent = oldDescendant.getParent(); + + if(parent!= null && parent.getType() == 32 && !alreadyAddParentMethodName){ //parent is method invocation statement + String parentLabel = parent.getLabel(); + String[] pns = parentLabel.split("\\." + methodName); + if(pns.length > 1) { + String parentName = pns[0]; + String[] parentNameSplit = parentName.split("@@"); + if (parentNameSplit.length > 1) { + String parentMethodName = parentNameSplit[1]; + String s1 = parentMethodName.split("@TO@")[0]; + oldTokens.add(s1.trim()); + alreadyAddParentMethodName = true; + } + } + + } oldTokens.add(methodName); }else if( sType.equals("59") || sType.equals("43")|| sType.equals("53") || sType.equals("7") || sType.equals("27") || sType.equals("83") || sType.equals("44") ||sType.equals("78") || sType.equals("41") || addToken){ // if(sType.equals("27") || sType.equals("83") || sType.equals("44")){ @@ -570,7 +602,7 @@ public class MultiThreadTreeLoaderCluster3 { return oldTokens; } - private static void coreCompare(String name , JedisPool jedisPool,String clusterName,JedisPool outerPool) { + private static void coreCompare(String name , JedisPool jedisPool,String clusterName,JedisPool outerPool,String type) { try (Jedis jedis = jedisPool.getResource()) { @@ -621,8 +653,8 @@ public class MultiThreadTreeLoaderCluster3 { // } try { - ITree oldTree = getTree(firstValue,outerPool); - ITree newTree = getTree(secondValue,outerPool); + ITree oldTree = getTree(firstValue,outerPool,type); + ITree newTree = getTree(secondValue,outerPool,type); if(oldTree == null || newTree == null) { jedis.select(0); diff --git a/src/main/java/edu/lu/uni/serval/Launcher.java b/src/main/java/edu/lu/uni/serval/Launcher.java index 976afd7..ef6292a 100644 --- a/src/main/java/edu/lu/uni/serval/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/Launcher.java @@ -46,11 +46,11 @@ public class Launcher { // inputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairs"; // gumInput = "/Users/anilkoyuncu/bugStudy/dataset/Defects4J/"; portInner = "6380"; - serverWait = "50000"; + serverWait = "5000"; chunkName = "Bug13April.txt.csv.rdb"; // dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; numOfWorkers = "10"; - jobType = "AKKA"; + jobType = "LEVEL1DB"; port = "6399"; // pairsPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImportDefects4J"; // gumOutput = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutputDefects4J"; @@ -97,23 +97,28 @@ public class Launcher { AkkaTreeLoader.main(portInner, serverWait, dbDir, chunk +"MOV"+".txt.csv.rdb", port, "MOV"+dumpsName); break; case "LEVEL1DB": - TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-BugsDotJar.rdb", dbDir + "/level1-BugsDotJar/"); + TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ "UPD"+".rdb", dbDir + "/level1-Bug13April/",pjName + "UPD"); + TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ "INS"+".rdb", dbDir + "/level1-Bug13April/",pjName + "INS"); + TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ "DEL"+".rdb", dbDir + "/level1-Bug13April/",pjName + "DEL"); + TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ "MOV"+".rdb", dbDir + "/level1-Bug13April/",pjName + "MOV"); break; //CALC python abstractPatch.py to from cluster folder case "L2CALCPAIRS": // MultiThreadTreeLoaderCluster.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterDefect4J","/Users/anilkoyuncu/bugStudy/dataset/"); - MultiThreadTreeLoaderCluster.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/cluster", datasetPath); + MultiThreadTreeLoaderCluster.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterBug13April", datasetPath); break; case "L2PAIRDB": // MultiThreadTreeLoaderCluster.mainCompare("6300","/Users/anilkoyuncu/bugStudy/dataset/pairs-csv","/Users/anilkoyuncu/bugStudy/dataset/redisSingleImport.sh",dbDir,"clusterl1-d4j.rdb",dumpsName,"6301"); - MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath + "/pairs-csv", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-13april.rdb", dumpsName, "6301"); + MultiThreadTreeLoaderCluster.mainCompare("6300", datasetPath + "/pairs-csv", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl1-13april.rdb", "UPD"+dumpsName, "6301",serverWait,"UPD"); break; + + //CALC via python case "L3CALCPAIRS": // MultiThreadTreeLoaderCluster3.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterDefect4J-2l",datasetPath); - MultiThreadTreeLoaderCluster3.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/cluster-2l", datasetPath); + MultiThreadTreeLoaderCluster3.calculatePairsOfClusters("/Users/anilkoyuncu/bugStudy/code/python/clusterBug13April-2l", datasetPath); break; case "L3PAIRDB": - MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath + "/pairs-2l-csv", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-13april.rdb", dumpsName, "6301"); + MultiThreadTreeLoaderCluster3.mainCompare("6300", datasetPath + "/pairs-2l-csv", datasetPath + "/redisSingleImport.sh", dbDir, "clusterl2-13april.rdb", "UPD"+dumpsName, "6301",serverWait,"UPD"); break; } } catch (Exception e) {