From bbf90adb2054c0fe06ea83cda5769d7d8b1237e2 Mon Sep 17 00:00:00 2001 From: fixminer Date: Thu, 2 Aug 2018 18:11:30 +0200 Subject: [PATCH] pattern extraction --- .../cluster/AkkaTreeLoader.java | 2 +- .../FixPatternParser/cluster/StoreFile.java | 24 +- src/main/java/edu/lu/uni/serval/Launcher.java | 9 +- .../ParseFixPatternWorker.java | 14 +- .../edu/lu/uni/serval/PatternExtractor.java | 294 ++++++++++++++++++ 5 files changed, 331 insertions(+), 12 deletions(-) create mode 100644 src/main/java/edu/lu/uni/serval/PatternExtractor.java diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java index c03416a..4119e3e 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/AkkaTreeLoader.java @@ -204,7 +204,7 @@ public class AkkaTreeLoader { List keysByValue = getKeysByValue(ASTNodeMap.map, astNodeType); if(keysByValue.size() != 1){ - log.error("Birden cok astnodemapmapping"); + log.error("More than 1"); } newType = keysByValue.get(0); if(actionSet.getParent() == null){ diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java index d32bc63..b55731c 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java @@ -2,6 +2,7 @@ package edu.lu.uni.serval.FixPatternParser.cluster; import edu.lu.uni.serval.FixPatternParser.violations.CallShell; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import org.apache.commons.lang3.ArrayUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -65,15 +66,32 @@ public class StoreFile { .filter(x -> !x.getName().startsWith(".")) .collect(Collectors.toList()); List workList = new ArrayList(); + File[] dumps; for (File pj : pjs) { String pjName = pj.getName(); File[] files = pj.listFiles(); Stream fileStream = Arrays.stream(files); - List fs = fileStream - .filter(x -> x.getName().startsWith(operation)) + List fs; + if (operation.equals("ALL")){ + fs= fileStream + .filter(x -> x.getName().startsWith("UPD") || + x.getName().startsWith("INS") || + x.getName().startsWith("DEL") || + x.getName().startsWith("MOV")) .collect(Collectors.toList()); + File[] files1 = fs.get(0).listFiles(); + File[] files2 = fs.get(1).listFiles(); + File[] files3 = fs.get(2).listFiles(); + File[] files4 = fs.get(3).listFiles(); + dumps = Stream.of(files1, files2, files3,files4).flatMap(Stream::of).toArray(File[]::new); + }else{ + fs = fileStream + .filter(x -> x.getName().startsWith(operation)) + .collect(Collectors.toList()); + dumps = fs.get(0).listFiles(); + } + - File[] dumps = fs.get(0).listFiles(); for (File f : dumps) { String name = f.getName(); diff --git a/src/main/java/edu/lu/uni/serval/Launcher.java b/src/main/java/edu/lu/uni/serval/Launcher.java index 5dea4a0..c67cc19 100644 --- a/src/main/java/edu/lu/uni/serval/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/Launcher.java @@ -37,7 +37,7 @@ public class Launcher { String datasetPath = appProps.getProperty("datasetPath","/Users/anilkoyuncu/bugStudy/dataset"); String pjName = appProps.getProperty("pjName","allDataset"); String dbNo = appProps.getProperty("dbNo","0"); - String actionType = appProps.getProperty("actionType","UPD"); + String actionType = appProps.getProperty("actionType","ALL"); String threshold = appProps.getProperty("threshold","1"); String parameters = String.format("\nportInner %s " + @@ -94,6 +94,13 @@ public class Launcher { level1(portInner, serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput); level2(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); level3(serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, dumpsName, gumInput); + break; + case "EXTRACTPATTERN": + PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,port,pythonPath,datasetPath,pjName,dbNo,actionType,threshold); + break; + case "GETPATTERN": + PatternExtractor.mainLaunch(portInner,serverWait,numOfWorkers,jobType,port,pythonPath,datasetPath,pjName,dbNo,actionType,threshold); + break; } } catch (Exception e) { e.printStackTrace(); diff --git a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java index ae9cc74..8f88ce3 100644 --- a/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java +++ b/src/main/java/edu/lu/uni/serval/MultipleThreadsParser/ParseFixPatternWorker.java @@ -158,15 +158,15 @@ public class ParseFixPatternWorker extends UntypedActor { sizes.setLength(0); tokens.setLength(0); - FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); +// FileHelper.outputToFile("OUTPUT/testingInfo_" + id + ".list", testingInfo, true); testingInfo.setLength(0); } - String statistic = "\nNullGumTreeResults: " + nullGumTreeResults + "\nNoSourceCodeChanges: " + noSourceCodeChanges + - "\nNoStatementChanges: " + noStatementChanges + "\nNullDiffEntry: " + nullDiffEntry + "\nNullMatchedGumTreeResults: " + nullMappingGumTreeResults + - "\nPureDeletion: " + pureDeletion + "\nLargeHunk: " + largeHunk + "\nNullSourceCode: " + nullSourceCode + - "\nTestingInfo: " + testInfos + "\nTimeout: " + timeouts; - FileHelper.outputToFile("OUTPUT/statistic_" + id + ".list", statistic, false); - FileHelper.outputToFile("OUTPUT/UnfixedV_" + id + ".list", builder, false); +// String statistic = "\nNullGumTreeResults: " + nullGumTreeResults + "\nNoSourceCodeChanges: " + noSourceCodeChanges + +// "\nNoStatementChanges: " + noStatementChanges + "\nNullDiffEntry: " + nullDiffEntry + "\nNullMatchedGumTreeResults: " + nullMappingGumTreeResults + +// "\nPureDeletion: " + pureDeletion + "\nLargeHunk: " + largeHunk + "\nNullSourceCode: " + nullSourceCode + +// "\nTestingInfo: " + testInfos + "\nTimeout: " + timeouts; +// FileHelper.outputToFile("OUTPUT/statistic_" + id + ".list", statistic, false); +// FileHelper.outputToFile("OUTPUT/UnfixedV_" + id + ".list", builder, false); log.info("Worker #" + id +"finialized parsing " + counter + " files..."); log.info("Worker #" + id + " finialized the work..."); diff --git a/src/main/java/edu/lu/uni/serval/PatternExtractor.java b/src/main/java/edu/lu/uni/serval/PatternExtractor.java new file mode 100644 index 0000000..4810bad --- /dev/null +++ b/src/main/java/edu/lu/uni/serval/PatternExtractor.java @@ -0,0 +1,294 @@ +package edu.lu.uni.serval; + +import com.github.gumtreediff.tree.ITree; +import com.github.gumtreediff.tree.TreeContext; +import edu.lu.uni.serval.FixPatternParser.violations.CallShell; +import edu.lu.uni.serval.FixPatternParser.violations.TestHunkParser; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; + +import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.getASTTree; +import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.getSimpliedTree; +import static edu.lu.uni.serval.Launcher.mainLaunch; + +/** + * Created by anilkoyuncu on 02/08/2018. + */ +public class PatternExtractor { + private static Logger log = LoggerFactory.getLogger(PatternExtractor.class); + + + public static void mainLaunch(String portInner,String serverWait, String numOfWorkers,String jobType,String port, String pythonPath, String datasetPath, String pjName, String dbNo, String actionType,String threshold) { + + + String dbDir; + String pairsPath; + String dumpsName; + String gumInput; + String gumOutput; + + gumInput = datasetPath + "/" + pjName + "/"; + gumOutput = datasetPath + "/EnhancedASTDiff" + pjName; + dbDir = datasetPath + "/redis"; + pairsPath = datasetPath + "/pairsImport" + pjName; + dumpsName = "dumps-" + pjName + ".rdb"; + + try { + + String rootPath = "/Users/anilkoyuncu/bugStudy/code/python"; + + String IDLIST_PATH =rootPath + "/defects4jpatterns.txt"; + + List fixes = readIdList(IDLIST_PATH); + + switch (jobType) { + case "EXTRACTPATTERN": + loadDB(gumOutput, portInner, serverWait, dbDir, actionType+dumpsName,actionType,fixes); + break; + case "GETPATTERN": + getPattern(fixes,actionType); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + + public static void loadDB(String inputPath,String portInner,String serverWait,String dbDir,String chunkName,String operation,List fixes) throws Exception { +// String inputPath; +// String portInner; +// String serverWait; +// String dbDir; +// String chunkName; +// String numOfWorkers; +// if (args.length > 0) { +// inputPath = args[0]; +// portInner = args[1]; +// serverWait = args[2]; +// chunkName = args[3]; +// numOfWorkers = args[4]; +// dbDir = args[5]; +// } else { +// inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; +// portInner = "6399"; +// serverWait = "10000"; +// chunkName ="dumps.rdb"; +// dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; +// numOfWorkers = "1"; +// } + String parameters = String.format("\nInput path %s \nportInner %s \nserverWait %s \nchunkName %s \ndbDir %s \noperation %s", inputPath, portInner, serverWait, chunkName, dbDir, operation); + log.info(parameters); + CallShell cs = new CallShell(); + String cmd = "bash " + dbDir + "/" + "startServer.sh" + " %s %s %s"; + cmd = String.format(cmd, dbDir, chunkName, Integer.valueOf(portInner)); +// loadRedis(cmd,serverWait); + cs.runShell(cmd, serverWait); + + JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000); + + String clusterPath = "/Users/anilkoyuncu/bugStudy/release/dataset/output/clusterallDatasetUPD/"; + String savePath = "/Users/anilkoyuncu/bugStudy/release/dataset/dumps/"; + for (String pattern:fixes) { + File folder = new File(clusterPath + pattern); + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List patches = stream + .filter(x -> !x.getName().startsWith(".")) + .filter(x-> x.getName().endsWith(".git")) + .collect(Collectors.toList()); + + for (File patch : patches) { + String fn = patch.getName(); + String[] split = fn.split("_"); + String project = split[split.length -1]; + List list = new ArrayList(Arrays.asList(split)); + list.remove(list.size() - 1); + String joinFN = String.join("_", list); + fn = project + "/" + operation + "/" + joinFN; + String saveFN = pattern + "_"+ project + "_" + operation + "_" + joinFN; + Jedis inner = null; + String s = null; + try { + inner = outerPool.getResource(); + s = inner.get(fn); + if (s != null) { + + HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(s); + + + BufferedWriter writer = new BufferedWriter(new FileWriter(savePath+saveFN)); + writer.write(toString(actionSet)); + + writer.close(); + + + + + + } + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (inner != null) { + inner.close(); + } + } + + + + + + + } + + } + + + + + + } + + public static void getPattern(List fixes,String operation){ + String clusterPath = "/Users/anilkoyuncu/bugStudy/release/dataset/output/clusterallDatasetUPD/"; + String savePath = "/Users/anilkoyuncu/bugStudy/release/dataset/dumps/"; + for (String pattern:fixes) { + File folder = new File(clusterPath + pattern); + File[] listOfFiles = folder.listFiles(); + Stream stream = Arrays.stream(listOfFiles); + List patches = stream + .filter(x -> !x.getName().startsWith(".")) + .filter(x -> x.getName().endsWith(".git")) + .collect(Collectors.toList()); + + for (File patch : patches) { + String fn = patch.getName(); + String[] split = fn.split("_"); + String project = split[split.length - 1]; + List list = new ArrayList(Arrays.asList(split)); + list.remove(list.size() - 1); + String joinFN = String.join("_", list); + fn = project + "/" + operation + "/" + joinFN; + String saveFN = pattern + "_" + project + "_" + operation + "_" + joinFN; + try{ + String content = new String(Files.readAllBytes(Paths.get(savePath + saveFN))); + HierarchicalActionSet actionSet = (HierarchicalActionSet) fromString(content); + + + ITree simpliedTree = getSimpliedTree(actionSet); + simpliedTree.toString(); + }catch (Exception e){ + e.printStackTrace(); + } + } + } + } + + public static ITree getSimpliedTree(HierarchicalActionSet actionSet) { + + ITree tree = null; + Jedis inner = null; + try { + + ITree parent = null; + ITree children = null; + TreeContext tc = new TreeContext(); + tree = getASTTree(actionSet, parent, children, tc); +// tree.setParent(null); + tc.validate(); + + } catch (Exception e) { + e.printStackTrace(); + } + return tree; + } + + + + static final JedisPoolConfig poolConfig = buildPoolConfig(); + + private static String toString( Serializable o ) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream( baos ); + oos.writeObject( o ); + oos.close(); + return Base64.getEncoder().encodeToString(baos.toByteArray()); + } + + /** Read the object from Base64 string. */ + private static Object fromString( String s ) throws IOException , + ClassNotFoundException { + byte[] data = Base64.getDecoder().decode(s); + ObjectInputStream ois = new ObjectInputStream( + new ByteArrayInputStream(data)); + Object o = ois.readObject(); + ois.close(); + return o; + } + private static JedisPoolConfig buildPoolConfig() { + final JedisPoolConfig poolConfig = new JedisPoolConfig(); + poolConfig.setMaxTotal(128); + poolConfig.setMaxIdle(128); + poolConfig.setMinIdle(16); + poolConfig.setTestOnBorrow(false); + poolConfig.setTestOnReturn(false); + poolConfig.setTestWhileIdle(true); + poolConfig.setMinEvictableIdleTimeMillis(Duration.ofMinutes(60).toMillis()); + poolConfig.setTimeBetweenEvictionRunsMillis(Duration.ofHours(30).toMillis()); +// poolConfig.setNumTestsPerEvictionRun(3); + poolConfig.setBlockWhenExhausted(true); + + return poolConfig; + } + + + public static List readIdList(String fileName) { + try { + BufferedReader reader = new BufferedReader(new FileReader(new File(fileName))); + + // n = Rownum ; m = Colnum + List fixCommits = new ArrayList(); + String line = reader.readLine(); + + int i = 0, j = 0; + while (line != null) { + String strArray[] = line.split("\t"); + + if (!line.trim().isEmpty()) { + + GitTravellerDefects4J.FixCommit test = new GitTravellerDefects4J.FixCommit(strArray[1],strArray[0]); + fixCommits.add(strArray[0]); +// for (String s : strArray) { +// if (!s.trim().isEmpty()) { +// FixCommit test = new FixCommit("",""); +// array[i][j++] = s; +// } +// } + line = reader.readLine(); +// i++; +// j = 0; + } + } + reader.close(); + return fixCommits; + } catch (IOException ex) { + System.out.println("Problems.."); + } + return null; + } +}