diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java index e72be9e..f906103 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java @@ -10,6 +10,8 @@ import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanResult; import java.io.*; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.util.List; import static edu.lu.uni.serval.FixPatternParser.cluster.TreeLoaderClusterL1.poolConfig; @@ -44,7 +46,7 @@ public class CalculatePairs { ScanParams sc = new ScanParams(); //150000000 - sc.count(150000000); + sc.count(1500000000); sc.match("*"); scan = outer.scan("0", sc); @@ -59,26 +61,9 @@ public class CalculatePairs { byte [] buf = new byte[0]; String line = null; try { - FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv"); - DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); - - - - for (int i = 0; i < result.size(); i++) { - for (int j = i + 1; j < result.size(); j++) { - - - - line = String.valueOf(i) +"," + String.valueOf(j) + "," + result.get(i) + "," + result.get(j)+"\n"; - outStream.write(line.getBytes()); - - } - } - outStream.close(); -// int fileCounter = 0; -// FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +pjName +String.valueOf(fileCounter)+".txt", "rw").getChannel(); -// int maxSize = 500*500000; -// ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, maxSize); +// FileOutputStream fos = new FileOutputStream(outputPath + "/" +pjName+".csv"); +// DataOutputStream outStream = new DataOutputStream(new BufferedOutputStream(fos)); +// // // // for (int i = 0; i < result.size(); i++) { @@ -86,23 +71,40 @@ public class CalculatePairs { // // // -// line = String.valueOf(i) +"\t" + String.valueOf(j) + "\t" + result.get(i) + "\t" + result.get(j)+"\n"; -// buf = line.getBytes(); -// if(wrBuf.remaining() > 500) { -// wrBuf.put(buf); -// }else{ -// log.info("Next pair dump"); -// fileCounter++; -// rwChannel = new RandomAccessFile(outputPath+"/" +pjName+String.valueOf(fileCounter)+".txt", "rw").getChannel(); -// wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, maxSize); -// } -// -// -// +// line = String.valueOf(i) +"," + String.valueOf(j) + "," + result.get(i) + "," + result.get(j)+"\n"; +// outStream.write(line.getBytes()); // // } // } -// rwChannel.close(); +// outStream.close(); + int fileCounter = 0; + FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +pjName +String.valueOf(fileCounter)+".txt", "rw").getChannel(); + int maxSize = 500*500000; + ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, maxSize); + + + for (int i = 0; i < result.size(); i++) { + for (int j = i + 1; j < result.size(); j++) { + + + + line = String.valueOf(i) +"\t" + String.valueOf(j) + "\t" + result.get(i) + "\t" + result.get(j)+"\n"; + buf = line.getBytes(); + if(wrBuf.remaining() > 500) { + wrBuf.put(buf); + }else{ + log.info("Next pair dump"); + fileCounter++; + rwChannel = new RandomAccessFile(outputPath+"/" +pjName+String.valueOf(fileCounter)+".txt", "rw").getChannel(); + wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, maxSize); + } + + + + + } + } + rwChannel.close(); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java index a170fe6..3571263 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/ImportPairs2DB.java @@ -32,7 +32,7 @@ public class ImportPairs2DB { File[] subFolders = folder.listFiles(); Stream stream = Arrays.stream(subFolders); List pjs = stream - .filter(x -> x.getName().endsWith(".csv")) + .filter(x -> x.getName().endsWith(".txt")) .collect(Collectors.toList()); Integer portInt = Integer.valueOf(portInner); diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java index b55731c..d89e87b 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/StoreFile.java @@ -72,7 +72,7 @@ public class StoreFile { File[] files = pj.listFiles(); Stream fileStream = Arrays.stream(files); List fs; - if (operation.equals("ALL")){ + if (operation.equals("ALLOP")){ fs= fileStream .filter(x -> x.getName().startsWith("UPD") || x.getName().startsWith("INS") || diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java index 445cd3a..80a8802 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/MultiThreadTreeLoaderCluster3.java @@ -565,6 +565,8 @@ public class MultiThreadTreeLoaderCluster3 { CharSequence[] oldSequences = oldTokens.toArray(new CharSequence[oldTokens.size()]); CharSequence[] newSequences = newTokens.toArray(new CharSequence[newTokens.size()]); JaroWinklerDistance jwd = new JaroWinklerDistance(); + LevenshteinDistance ld = new LevenshteinDistance(); + Double overallSimi = Double.valueOf(1); if(oldSequences.length > 0 && (oldSequences.length == newSequences.length)){ for (int idx = 0; idx < newSequences.length; idx++) { diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParser.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParser.java index d9bdd1f..3d8916e 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParser.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/violations/TestHunkParser.java @@ -55,6 +55,7 @@ public class TestHunkParser { FileHelper.createDirectory(GUM_TREE_OUTPUT + "/INS"); FileHelper.createDirectory(GUM_TREE_OUTPUT + "/DEL"); FileHelper.createDirectory(GUM_TREE_OUTPUT + "/MOV"); + FileHelper.createDirectory(GUM_TREE_OUTPUT + "/ALL"); // FileHelper.deleteDirectory(editScriptsFilePath); // FileHelper.deleteDirectory(patchesSourceCodeFilePath); // FileHelper.deleteDirectory(buggyTokensFilePath); diff --git a/src/main/java/edu/lu/uni/serval/Launcher.java b/src/main/java/edu/lu/uni/serval/Launcher.java index 9e167fc..4944628 100644 --- a/src/main/java/edu/lu/uni/serval/Launcher.java +++ b/src/main/java/edu/lu/uni/serval/Launcher.java @@ -77,6 +77,13 @@ public class Launcher { case "CACHE": StoreFile.main(gumOutput, portInner, serverWait, dbDir, actionType+dumpsName,actionType); break; + case "COMP": + CalculatePairs.main(serverWait, dbDir, actionType+dumpsName, portInner, pairsPath+actionType, pjName+actionType); + + ImportPairs2DB.main(pairsPath+actionType, portInner, serverWait, dbDir,datasetPath); + + AkkaTreeLoader.main(portInner, serverWait, dbDir, pjName +actionType+".csv.rdb" , port, actionType+dumpsName); + case "LEVEL1": level1(portInner, serverWait, port, pythonPath, datasetPath, pjName, actionType, threshold, dbDir, pairsPath, dumpsName, gumInput); break; @@ -112,13 +119,9 @@ public class Launcher { } private static void level1(String portInner, String serverWait, String port, String pythonPath, String datasetPath, String pjName, String actionType, String threshold, String dbDir, String pairsPath, String dumpsName, String gumInput) throws Exception { -// CalculatePairs.main(serverWait, dbDir, actionType+dumpsName, portInner, pairsPath+actionType, pjName+actionType); -// -// ImportPairs2DB.main(pairsPath+actionType, portInner, serverWait, dbDir,datasetPath); -// -// AkkaTreeLoader.main(portInner, serverWait, dbDir, pjName +actionType+".csv.rdb" , port, actionType+dumpsName); -// -// TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ actionType+".rdb", dbDir ,pjName + actionType); + + + TreeLoaderClusterL1.main(portInner, serverWait, port, dbDir, "level1-"+pjName+ actionType+".rdb", dbDir ,pjName + actionType); CallShell cs1 =new CallShell(); String db1 = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";