diff --git a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java index 5e9e693..500445e 100644 --- a/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java +++ b/src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java @@ -1,5 +1,8 @@ package edu.lu.uni.serval.FixPatternParser.cluster; +import com.github.gumtreediff.tree.ITree; +import edu.lu.uni.serval.utils.FileHelper; +import org.javatuples.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -8,9 +11,14 @@ import redis.clients.jedis.ScanParams; import redis.clients.jedis.ScanResult; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.StandardCopyOption; +import java.util.ArrayList; import java.util.List; import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; @@ -30,6 +38,7 @@ public class CalculatePairs { String serverWait; String dbDir; String chunkName; + String outputPath; if (args.length > 0) { inputPath = args[0]; @@ -39,6 +48,7 @@ public class CalculatePairs { dbDir = args[5]; port = args[6]; + outputPath = args[7]; } else { inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; @@ -46,7 +56,7 @@ public class CalculatePairs { port = "6399"; portInner = "6380"; serverWait = "10000"; - + outputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImport"; chunkName ="chunk"; dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; @@ -60,13 +70,10 @@ public class CalculatePairs { cmd = String.format(cmd, dbDir,"dumps.rdb",Integer.valueOf(port)); loadRedis(cmd,serverWait); - String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; - cmd = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner)); - loadRedis(cmd,serverWait); + FileHelper.createDirectory(outputPath); + JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); - JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000); - ScanResult scan; try (Jedis outer = outerPool.getResource()) { @@ -85,70 +92,128 @@ public class CalculatePairs { } List result = scan.getResult(); - int fileCounter = 0; - int pairCounter = 0; - for (int i = 0; i < result.size(); i++) { - for (int j = i + 1; j < result.size(); j++) { - Jedis jedis = null; - String key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j); - try { - jedis = innerPool.getResource(); - key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j); -// String value = treesFileNames.get(i).split("GumTreeOutput2")[1] +","+treesFileNames.get(j).split("GumTreeOutput2")[1]; -// jedis.set(key,value); - - jedis.hset(key, "0", result.get(i)); - jedis.hset(key, "1", result.get(j)); - pairCounter ++; - - //10000000 - if (pairCounter % 10000000 == 0) { - - File dbPath = new File(dbDir + "/" + chunkName); - File savePath = new File(dbDir + "/" + "chunk" + String.valueOf(fileCounter) + ".rdb"); - try { - jedis.save(); - log.info("saving key {} chunk {}",key,fileCounter); - while (jedis.ping() == "PONG") { - log.info("wait"); - } - Thread.sleep(Integer.valueOf(serverWait)); - - Files.copy(dbPath.toPath(), savePath.toPath(), StandardCopyOption.REPLACE_EXISTING); - fileCounter++; - jedis.flushDB(); - while (jedis.ping() == "PONG") { - log.info("wait"); - } - Thread.sleep(Integer.valueOf(serverWait)); - - } catch (IOException e) { - - e.printStackTrace(); - } + + byte [] buf = new byte[0]; + String line = null; + try { + + FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +"textfile.txt", "rw").getChannel(); + ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE); + int fileCounter = 0; + + + for (int i = 0; i < result.size(); i++) { + for (int j = i + 1; j < result.size(); j++) { + + + + line = String.valueOf(i) +"\t" + String.valueOf(j) + "\t" + result.get(i) + "\t" + result.get(j)+"\n"; + buf = line.getBytes(); + if(wrBuf.remaining() > 500) { + wrBuf.put(buf); + }else{ + log.info("Next pair dump"); + fileCounter++; + rwChannel = new RandomAccessFile(outputPath+"/" +"textfile"+String.valueOf(fileCounter)+".txt", "rw").getChannel(); + wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE); } - }catch (Exception e) { - log.error(e.toString() + " {}", (key)); - }finally { - if (jedis != null) { - jedis.close(); + + } } - } + rwChannel.close(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + }catch (java.nio.BufferOverflowException e) { + log.error(line); + log.error(String.valueOf(buf.length)); + e.printStackTrace(); } + log.info("Done pairs"); + } + + + + + // comparePairs(inputPath, innerPool,outerPool, serverWait,chunkName,dbDir,numOfWorkers); - String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; - stopServer = String.format(stopServer,Integer.valueOf(portInner)); - loadRedis(stopServer,serverWait); +// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; +// stopServer = String.format(stopServer,Integer.valueOf(portInner)); +// loadRedis(stopServer,serverWait); // } - } + + + +// public static void corePairs(,ArrayList> list){ +// String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; +// cmd = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner)); +// loadRedis(cmd,serverWait); +// JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000); +// +// +// +// Jedis jedis = null; +// for (Pair objects : list) { +// +// try { +// String key = objects.getValue0(); +// String value = objects.getValue1(); +// jedis = innerPool.getResource(); +// +// +// String[] split = value.split(","); +// +// +// jedis.hset(key, "0", split[0]); +// jedis.hset(key, "1", split[1]); +// +// +// //10000000 +// if (pairCounter % 10000000 == 0) { +// +// File dbPath = new File(dbDir + "/" + chunkName); +// File savePath = new File(dbDir + "/" + "chunk" + String.valueOf(fileCounter) + ".rdb"); +// try { +// jedis.save(); +// log.info("saving key {} chunk {}",key,fileCounter); +// while (jedis.ping() == "PONG") { +// log.info("wait"); +// } +// Thread.sleep(Integer.valueOf(serverWait)); +// +// Files.copy(dbPath.toPath(), savePath.toPath(), StandardCopyOption.REPLACE_EXISTING); +// fileCounter++; +// jedis.flushDB(); +// while (jedis.ping() == "PONG") { +// log.info("wait"); +// } +// Thread.sleep(Integer.valueOf(serverWait)); +// +// } catch (IOException e) { +// +// e.printStackTrace(); +// } +// +// +// } +// }catch (Exception e) { +// log.error(e.toString() + " {}", (key)); +// }finally { +// if (jedis != null) { +// jedis.close(); +// } +// } +// } +// } }