file
This commit is contained in:
Anil Koyuncu
@@ -1,5 +1,8 @@
|
||||
package edu.lu.uni.serval.FixPatternParser.cluster;
|
||||
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
import edu.lu.uni.serval.utils.FileHelper;
|
||||
import org.javatuples.Pair;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import redis.clients.jedis.Jedis;
|
||||
@@ -8,9 +11,14 @@ import redis.clients.jedis.ScanParams;
|
||||
import redis.clients.jedis.ScanResult;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis;
|
||||
@@ -30,6 +38,7 @@ public class CalculatePairs {
|
||||
String serverWait;
|
||||
String dbDir;
|
||||
String chunkName;
|
||||
String outputPath;
|
||||
|
||||
if (args.length > 0) {
|
||||
inputPath = args[0];
|
||||
@@ -39,6 +48,7 @@ public class CalculatePairs {
|
||||
|
||||
dbDir = args[5];
|
||||
port = args[6];
|
||||
outputPath = args[7];
|
||||
|
||||
} else {
|
||||
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
|
||||
@@ -46,7 +56,7 @@ public class CalculatePairs {
|
||||
port = "6399";
|
||||
portInner = "6380";
|
||||
serverWait = "10000";
|
||||
|
||||
outputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImport";
|
||||
chunkName ="chunk";
|
||||
dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
|
||||
|
||||
@@ -60,13 +70,10 @@ public class CalculatePairs {
|
||||
cmd = String.format(cmd, dbDir,"dumps.rdb",Integer.valueOf(port));
|
||||
loadRedis(cmd,serverWait);
|
||||
|
||||
String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
|
||||
cmd = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner));
|
||||
loadRedis(cmd,serverWait);
|
||||
FileHelper.createDirectory(outputPath);
|
||||
|
||||
|
||||
JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
|
||||
JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000);
|
||||
|
||||
|
||||
ScanResult<String> scan;
|
||||
try (Jedis outer = outerPool.getResource()) {
|
||||
@@ -85,70 +92,128 @@ public class CalculatePairs {
|
||||
}
|
||||
List<String> result = scan.getResult();
|
||||
|
||||
int fileCounter = 0;
|
||||
int pairCounter = 0;
|
||||
for (int i = 0; i < result.size(); i++) {
|
||||
for (int j = i + 1; j < result.size(); j++) {
|
||||
Jedis jedis = null;
|
||||
String key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j);
|
||||
try {
|
||||
jedis = innerPool.getResource();
|
||||
key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j);
|
||||
// String value = treesFileNames.get(i).split("GumTreeOutput2")[1] +","+treesFileNames.get(j).split("GumTreeOutput2")[1];
|
||||
// jedis.set(key,value);
|
||||
|
||||
jedis.hset(key, "0", result.get(i));
|
||||
jedis.hset(key, "1", result.get(j));
|
||||
pairCounter ++;
|
||||
|
||||
//10000000
|
||||
if (pairCounter % 10000000 == 0) {
|
||||
|
||||
File dbPath = new File(dbDir + "/" + chunkName);
|
||||
File savePath = new File(dbDir + "/" + "chunk" + String.valueOf(fileCounter) + ".rdb");
|
||||
try {
|
||||
jedis.save();
|
||||
log.info("saving key {} chunk {}",key,fileCounter);
|
||||
while (jedis.ping() == "PONG") {
|
||||
log.info("wait");
|
||||
}
|
||||
Thread.sleep(Integer.valueOf(serverWait));
|
||||
|
||||
Files.copy(dbPath.toPath(), savePath.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
fileCounter++;
|
||||
jedis.flushDB();
|
||||
while (jedis.ping() == "PONG") {
|
||||
log.info("wait");
|
||||
}
|
||||
Thread.sleep(Integer.valueOf(serverWait));
|
||||
|
||||
} catch (IOException e) {
|
||||
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
|
||||
byte [] buf = new byte[0];
|
||||
String line = null;
|
||||
try {
|
||||
|
||||
FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +"textfile.txt", "rw").getChannel();
|
||||
ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE);
|
||||
int fileCounter = 0;
|
||||
|
||||
|
||||
for (int i = 0; i < result.size(); i++) {
|
||||
for (int j = i + 1; j < result.size(); j++) {
|
||||
|
||||
|
||||
|
||||
line = String.valueOf(i) +"\t" + String.valueOf(j) + "\t" + result.get(i) + "\t" + result.get(j)+"\n";
|
||||
buf = line.getBytes();
|
||||
if(wrBuf.remaining() > 500) {
|
||||
wrBuf.put(buf);
|
||||
}else{
|
||||
log.info("Next pair dump");
|
||||
fileCounter++;
|
||||
rwChannel = new RandomAccessFile(outputPath+"/" +"textfile"+String.valueOf(fileCounter)+".txt", "rw").getChannel();
|
||||
wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE);
|
||||
}
|
||||
}catch (Exception e) {
|
||||
log.error(e.toString() + " {}", (key));
|
||||
|
||||
|
||||
}finally {
|
||||
if (jedis != null) {
|
||||
jedis.close();
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
rwChannel.close();
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}catch (java.nio.BufferOverflowException e) {
|
||||
log.error(line);
|
||||
log.error(String.valueOf(buf.length));
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
log.info("Done pairs");
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// comparePairs(inputPath, innerPool,outerPool, serverWait,chunkName,dbDir,numOfWorkers);
|
||||
|
||||
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
|
||||
stopServer = String.format(stopServer,Integer.valueOf(portInner));
|
||||
loadRedis(stopServer,serverWait);
|
||||
// String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
|
||||
// stopServer = String.format(stopServer,Integer.valueOf(portInner));
|
||||
// loadRedis(stopServer,serverWait);
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
// public static void corePairs(,ArrayList<Pair<String,String>> list){
|
||||
// String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
|
||||
// cmd = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner));
|
||||
// loadRedis(cmd,serverWait);
|
||||
// JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000);
|
||||
//
|
||||
//
|
||||
//
|
||||
// Jedis jedis = null;
|
||||
// for (Pair<String, String> objects : list) {
|
||||
//
|
||||
// try {
|
||||
// String key = objects.getValue0();
|
||||
// String value = objects.getValue1();
|
||||
// jedis = innerPool.getResource();
|
||||
//
|
||||
//
|
||||
// String[] split = value.split(",");
|
||||
//
|
||||
//
|
||||
// jedis.hset(key, "0", split[0]);
|
||||
// jedis.hset(key, "1", split[1]);
|
||||
//
|
||||
//
|
||||
// //10000000
|
||||
// if (pairCounter % 10000000 == 0) {
|
||||
//
|
||||
// File dbPath = new File(dbDir + "/" + chunkName);
|
||||
// File savePath = new File(dbDir + "/" + "chunk" + String.valueOf(fileCounter) + ".rdb");
|
||||
// try {
|
||||
// jedis.save();
|
||||
// log.info("saving key {} chunk {}",key,fileCounter);
|
||||
// while (jedis.ping() == "PONG") {
|
||||
// log.info("wait");
|
||||
// }
|
||||
// Thread.sleep(Integer.valueOf(serverWait));
|
||||
//
|
||||
// Files.copy(dbPath.toPath(), savePath.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
// fileCounter++;
|
||||
// jedis.flushDB();
|
||||
// while (jedis.ping() == "PONG") {
|
||||
// log.info("wait");
|
||||
// }
|
||||
// Thread.sleep(Integer.valueOf(serverWait));
|
||||
//
|
||||
// } catch (IOException e) {
|
||||
//
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
//
|
||||
//
|
||||
// }
|
||||
// }catch (Exception e) {
|
||||
// log.error(e.toString() + " {}", (key));
|
||||
// }finally {
|
||||
// if (jedis != null) {
|
||||
// jedis.close();
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user