azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

file

Browse Source
This commit is contained in:
Anil Koyuncu
2018-04-05 15:54:11 +02:00
parent 655ca03374
commit cc3fd2af12
1 changed files with 122 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/FixPatternParser/cluster/CalculatePairs.java
+122 -57
View File
@@ -1,5 +1,8 @@
package edu.lu.uni.serval.FixPatternParser.cluster; package edu.lu.uni.serval.FixPatternParser.cluster;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.utils.FileHelper;
import org.javatuples.Pair;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;
@@ -8,9 +11,14 @@ import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult; import redis.clients.jedis.ScanResult;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.StandardCopyOption; import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis; import static edu.lu.uni.serval.FixPatternParser.cluster.AkkaTreeLoader.loadRedis;
@@ -30,6 +38,7 @@ public class CalculatePairs {
String serverWait; String serverWait;
String dbDir; String dbDir;
String chunkName; String chunkName;
String outputPath;
if (args.length > 0) { if (args.length > 0) {
inputPath = args[0]; inputPath = args[0];
@@ -39,6 +48,7 @@ public class CalculatePairs {
dbDir = args[5]; dbDir = args[5];
port = args[6]; port = args[6];
outputPath = args[7];
} else { } else {
inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2"; inputPath = "/Users/anilkoyuncu/bugStudy/dataset/GumTreeOutput2";
@@ -46,7 +56,7 @@ public class CalculatePairs {
port = "6399"; port = "6399";
portInner = "6380"; portInner = "6380";
serverWait = "10000"; serverWait = "10000";
outputPath = "/Users/anilkoyuncu/bugStudy/dataset/pairsImport";
chunkName ="chunk"; chunkName ="chunk";
dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis"; dbDir = "/Users/anilkoyuncu/bugStudy/dataset/redis";
@@ -60,13 +70,10 @@ public class CalculatePairs {
cmd = String.format(cmd, dbDir,"dumps.rdb",Integer.valueOf(port)); cmd = String.format(cmd, dbDir,"dumps.rdb",Integer.valueOf(port));
loadRedis(cmd,serverWait); loadRedis(cmd,serverWait);
String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s"; FileHelper.createDirectory(outputPath);
cmd = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner));
loadRedis(cmd,serverWait);
JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000); JedisPool outerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(port),20000000);
JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000);
ScanResult<String> scan; ScanResult<String> scan;
try (Jedis outer = outerPool.getResource()) { try (Jedis outer = outerPool.getResource()) {
@@ -85,70 +92,128 @@ public class CalculatePairs {
} }
List<String> result = scan.getResult(); List<String> result = scan.getResult();
int fileCounter = 0;
int pairCounter = 0;
for (int i = 0; i < result.size(); i++) {
for (int j = i + 1; j < result.size(); j++) {
Jedis jedis = null;
String key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j);
try {
jedis = innerPool.getResource();
key = "pair_" + String.valueOf(i) + "_" + String.valueOf(j);
// String value = treesFileNames.get(i).split("GumTreeOutput2")[1] +","+treesFileNames.get(j).split("GumTreeOutput2")[1];
// jedis.set(key,value);
jedis.hset(key, "0", result.get(i));
jedis.hset(key, "1", result.get(j));
pairCounter ++;
//10000000
if (pairCounter % 10000000 == 0) {
File dbPath = new File(dbDir + "/" + chunkName);
File savePath = new File(dbDir + "/" + "chunk" + String.valueOf(fileCounter) + ".rdb");
try {
jedis.save();
log.info("saving key {} chunk {}",key,fileCounter);
while (jedis.ping() == "PONG") {
log.info("wait");
}
Thread.sleep(Integer.valueOf(serverWait));
Files.copy(dbPath.toPath(), savePath.toPath(), StandardCopyOption.REPLACE_EXISTING);
fileCounter++;
jedis.flushDB();
while (jedis.ping() == "PONG") {
log.info("wait");
}
Thread.sleep(Integer.valueOf(serverWait));
} catch (IOException e) {
e.printStackTrace();
}
byte [] buf = new byte[0];
String line = null;
try {
FileChannel rwChannel = new RandomAccessFile(outputPath + "/" +"textfile.txt", "rw").getChannel();
ByteBuffer wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE);
int fileCounter = 0;
for (int i = 0; i < result.size(); i++) {
for (int j = i + 1; j < result.size(); j++) {
line = String.valueOf(i) +"\t" + String.valueOf(j) + "\t" + result.get(i) + "\t" + result.get(j)+"\n";
buf = line.getBytes();
if(wrBuf.remaining() > 500) {
wrBuf.put(buf);
}else{
log.info("Next pair dump");
fileCounter++;
rwChannel = new RandomAccessFile(outputPath+"/" +"textfile"+String.valueOf(fileCounter)+".txt", "rw").getChannel();
wrBuf = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, Integer.MAX_VALUE);
} }
}catch (Exception e) {
log.error(e.toString() + " {}", (key));
}finally {
if (jedis != null) {
jedis.close();
} }
} }
} rwChannel.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}catch (java.nio.BufferOverflowException e) {
log.error(line);
log.error(String.valueOf(buf.length));
e.printStackTrace();
} }
log.info("Done pairs");
}
// comparePairs(inputPath, innerPool,outerPool, serverWait,chunkName,dbDir,numOfWorkers); // comparePairs(inputPath, innerPool,outerPool, serverWait,chunkName,dbDir,numOfWorkers);
String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s"; // String stopServer = "bash "+dbDir + "/" + "stopServer.sh" +" %s";
stopServer = String.format(stopServer,Integer.valueOf(portInner)); // stopServer = String.format(stopServer,Integer.valueOf(portInner));
loadRedis(stopServer,serverWait); // loadRedis(stopServer,serverWait);
// } // }
}
// public static void corePairs(,ArrayList<Pair<String,String>> list){
// String cmdInner = "bash "+dbDir + "/" + "startServer.sh" +" %s %s %s";
// cmd = String.format(cmdInner, dbDir,chunkName,Integer.valueOf(portInner));
// loadRedis(cmd,serverWait);
// JedisPool innerPool = new JedisPool(poolConfig, "127.0.0.1",Integer.valueOf(portInner),20000000);
//
//
//
// Jedis jedis = null;
// for (Pair<String, String> objects : list) {
//
// try {
// String key = objects.getValue0();
// String value = objects.getValue1();
// jedis = innerPool.getResource();
//
//
// String[] split = value.split(",");
//
//
// jedis.hset(key, "0", split[0]);
// jedis.hset(key, "1", split[1]);
//
//
// //10000000
// if (pairCounter % 10000000 == 0) {
//
// File dbPath = new File(dbDir + "/" + chunkName);
// File savePath = new File(dbDir + "/" + "chunk" + String.valueOf(fileCounter) + ".rdb");
// try {
// jedis.save();
// log.info("saving key {} chunk {}",key,fileCounter);
// while (jedis.ping() == "PONG") {
// log.info("wait");
// }
// Thread.sleep(Integer.valueOf(serverWait));
//
// Files.copy(dbPath.toPath(), savePath.toPath(), StandardCopyOption.REPLACE_EXISTING);
// fileCounter++;
// jedis.flushDB();
// while (jedis.ping() == "PONG") {
// log.info("wait");
// }
// Thread.sleep(Integer.valueOf(serverWait));
//
// } catch (IOException e) {
//
// e.printStackTrace();
// }
//
//
// }
// }catch (Exception e) {
// log.error(e.toString() + " {}", (key));
// }finally {
// if (jedis != null) {
// jedis.close();
// }
// }
// }
// }
} }