changes for jar

This commit is contained in:
fixminer
2020-04-07 18:06:53 +02:00
parent c5463f91f8
commit 40a8b0c290
57 changed files with 1180 additions and 301 deletions
+150
View File
@@ -0,0 +1,150 @@
package edu.lu.uni.serval;
import edu.lu.uni.serval.richedit.jobs.CompareTrees;
import edu.lu.uni.serval.richedit.jobs.EnhancedASTDiff;
import edu.lu.uni.serval.utils.ClusterToPattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Map;
import java.util.Properties;
/**
* Created by anilkoyuncu on 14/04/2018.
*/
public class Launcher {
private static Logger log = LoggerFactory.getLogger(Launcher.class);
public static void main(String[] args) throws IOException {
Properties appProps = new Properties();
// String hostname = "Unknown";
// try
// {
// InetAddress addr;
// addr = InetAddress.getLocalHost();
// hostname = addr.getHostName();
// }
// catch (UnknownHostException ex)
// {
// System.out.println("Hostname can not be resolved");
// }
// String appConfigPath;
// if (hostname.equals("Unknown")){
// appConfigPath = "src/main/resource/app.properties";
// }
// else{
// appConfigPath = "src/main/resource/"+hostname.split("\\.")[0]+".app.properties";
// }
if(args.length != 2)
{
System.out.println("Proper Usage is: \n\tfirst argument full path to .properties file (e.g. an example is located under resources) \n\tsecond argument jobType (e.g RICHEDITSCRIPT, COMPARE)");
System.exit(0);
}
String appConfigPath = args[0];
Yaml yaml = new Yaml();
// InputStream inputStream = this.getClass()
// .getClassLoader()
// .getResourceAsStream("customer.yaml");
Map<String, Object> obj = yaml.load(new FileInputStream(appConfigPath));
appProps.load(new FileInputStream(appConfigPath));
Map<String, Object> fixminer = (Map<String, Object>) obj.get("fixminer");
// String numOfWorkers = appProps.getProperty("numOfWorkers", "10");
String numOfWorkers = String.valueOf(fixminer.get("numOfWorkers"));
String portDumps = String.valueOf(fixminer.get("portDumps"));
String projectType = (String) fixminer.get("projectType");
String hunkLimit = String.valueOf(fixminer.get("hunkLimit"));
String patchSize = String.valueOf(fixminer.get("patchSize"));
String projectL = (String) fixminer.get("projectList");
String[] projectList = projectL.split(",");
String input = (String) fixminer.get("inputPath");
String redisPath = (String) fixminer.get("redisPath");
String srcMLPath = (String) fixminer.get("srcMLPath");
// String parameter = args[2];
String parameter = "L1";
String jobType = args[1];
// String jobType = "RICHEDITSCRIPT";
// String jobType = "COMPARE";
mainLaunch( numOfWorkers, jobType, portDumps,projectType,input,redisPath,parameter, srcMLPath,hunkLimit,projectList,patchSize);
}
public static void mainLaunch(String numOfWorkers, String jobType, String portDumps, String projectType, String input, String redisPath,String parameter,String srcMLPath,String hunkLimit,String[] projectList,String patchSize){
String dbDir;
String dumpsName;
String gumInput;
dumpsName = "dumps-"+projectType+".rdb";
gumInput = input;
dbDir = redisPath;
try {
switch (jobType) {
case "RICHEDITSCRIPT":
EnhancedASTDiff.main(gumInput, portDumps, dbDir, dumpsName, srcMLPath,parameter,hunkLimit,projectList,patchSize,projectType);
break;
case "COMPARE":
String job;
String compareDBName;
switch (parameter){
case "L1":
// job = "shape";
job = "single";
compareDBName = "clusterl0-gumInputALL.rdb";
break;
case "L2":
job = "action";
compareDBName = "clusterl1-gumInputALL.rdb";
break;
case "L3":
job = "token";
compareDBName = "clusterl2-gumInputALL.rdb";
break;
default:
throw new Error("unknown level please specify L1,L2,L3");
}
CompareTrees.main(redisPath, portDumps,dumpsName, job,numOfWorkers);
break;
case "PATTERN":
ClusterToPattern.main(portDumps,redisPath, dumpsName, parameter);
break;
default:
throw new Error("unknown Job");
}
} catch (Exception e) {
log.error(e.getMessage());
// e.printStackTrace();
}
}
}
@@ -1,7 +1,7 @@
package edu.lu.uni.serval.fixminer;
package edu.lu.uni.serval.richedit;
import edu.lu.uni.serval.fixminer.jobs.CompareTrees;
import edu.lu.uni.serval.fixminer.jobs.EnhancedASTDiff;
import edu.lu.uni.serval.richedit.jobs.CompareTrees;
import edu.lu.uni.serval.richedit.jobs.EnhancedASTDiff;
import edu.lu.uni.serval.utils.ClusterToPattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -15,9 +15,9 @@ import java.util.Properties;
/**
* Created by anilkoyuncu on 14/04/2018.
*/
public class Launcher {
public class Launcher2 {
private static Logger log = LoggerFactory.getLogger(Launcher.class);
private static Logger log = LoggerFactory.getLogger(Launcher2.class);
public static void main(String[] args) throws IOException {
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
/**
* Created by anilkoyuncu on 18/09/2018.
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
import akka.actor.ActorRef;
import akka.actor.Props;
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.utils.EDiffHelper;
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
import redis.clients.jedis.JedisPool;
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.gen.srcml.GumTreeCComparer;
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
import akka.actor.Props;
import akka.actor.UntypedActor;
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.tree.ITree;
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
import com.github.gumtreediff.actions.model.*;
import com.github.gumtreediff.tree.ITree;
@@ -1,7 +1,6 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.actions.model.*;
import com.github.gumtreediff.gen.srcml.NodeMap_new;
import com.github.gumtreediff.tree.ITree;
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
import java.io.File;
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
import redis.clients.jedis.JedisPool;
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
public interface ParserInterface {
@@ -1,4 +1,4 @@
package edu.lu.uni.serval.fixminer.ediff;
package edu.lu.uni.serval.richedit.ediff;
import redis.clients.jedis.JedisPool;
@@ -1,6 +1,5 @@
package edu.lu.uni.serval.fixminer.jobs;
package edu.lu.uni.serval.richedit.jobs;
import edu.lu.uni.serval.fixminer.ediff.EDiffHunkParser;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
@@ -31,7 +30,7 @@ public class CompareTrees {
public static void main(String redisPath, String portDumps, String dumpsName, String job,String numOfWorkers) throws Exception {
// shape /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis ALLdumps-gumInput.rdb clusterl0-gumInputALL.rdb /Users/anil.koyuncu/projects/test/fixminer-core/python/data/richEditScript
// shape /Users/anil.koyuncu/projects/test/richedit-core/python/data/redis ALLdumps-gumInput.rdb clusterl0-gumInputALL.rdb /Users/anil.koyuncu/projects/test/richedit-core/python/data/richEditScript
// String portInner = "6380";
String port = portDumps; //"6399";
@@ -1,8 +1,8 @@
package edu.lu.uni.serval.fixminer.jobs;
package edu.lu.uni.serval.richedit.jobs;
import edu.lu.uni.serval.fixminer.ediff.EDiffHunkParser;
import edu.lu.uni.serval.richedit.ediff.EDiffHunkParser;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.fixminer.ediff.MessageFile;
import edu.lu.uni.serval.richedit.ediff.MessageFile;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.PoolBuilder;
import me.tongfei.progressbar.ProgressBar;
@@ -47,6 +47,9 @@ public class EnhancedASTDiff {
}
File folder = new File(inputPath);
File[] listOfFiles = folder.listFiles();
if(listOfFiles == null){
throw new Exception("No projects found, please verify the projects in the input path");
}
Stream<File> stream = Arrays.stream(listOfFiles);
List<File> folders;
if (projectList.length == 1 && projectList[0].equals("")){
@@ -5,7 +5,7 @@ import com.github.gumtreediff.gen.srcml.NodeMap_new;
import com.github.gumtreediff.tree.ITree;
import com.github.gumtreediff.tree.TreeContext;
import com.github.gumtreediff.tree.TreeUtils;
import edu.lu.uni.serval.fixminer.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.richedit.ediff.HierarchicalActionSet;
import org.apache.commons.lang3.SerializationUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -8,10 +8,10 @@ projectList =
#projectList = libtiff,php-src,cpython,wireshark,gzip,gmp,lighttpd1.4,lighttpd2
#inputPath = /Users/anilkoyuncu/projects/gumInputLinux
#inputPath = /Users/anil.koyuncu/projects/fixminer/fixminer-data/gumInputLinux
#inputPath = /Users/anil.koyuncu/projects/richedit/richedit-data/gumInputLinux
inputPath = /Users/anil.koyuncu/projects/test/fixminer-data/patches
#redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
#redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
#redisPath = /Users/anil.koyuncu/projects/richedit/richedit-core/python/data/redis
#redisPath = /Users/anil.koyuncu/projects/richedit/richedit-core/python/data/redis
redisPath = /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis
#srcMLPath= /Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml
srcMLPath= /Users/anil.koyuncu/projects/test/srcML/bin/srcml
+3 -3
View File
@@ -9,10 +9,10 @@ patchSize = 50
projectList = libtiff,php-src,cpython,wireshark,gzip,gmp,lighttpd1.4,lighttpd2
#inputPath = /Users/anilkoyuncu/projects/gumInputLinux
#inputPath = /Users/anil.koyuncu/projects/fixminer/fixminer-data/gumInputLinux
#inputPath = /Users/anil.koyuncu/projects/richedit/richedit-data/gumInputLinux
inputPath = /Users/anil.koyuncu/projects/test/fixminer-data/patches
#redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
#redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
#redisPath = /Users/anil.koyuncu/projects/richedit/richedit-core/python/data/redis
#redisPath = /Users/anil.koyuncu/projects/richedit/richedit-core/python/data/redis
redisPath = /Users/anil.koyuncu/projects/test/fixminer-core/python/data/redis
#srcMLPath= /Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml
srcMLPath= /usr/local/bin/srcml
@@ -9,7 +9,7 @@ patchSize = 50
projectList = codeflaws
#inputPath = /Users/anilkoyuncu/projects/gumInputLinux
inputPath = /Users/anilkoyuncu/projects/fixminer/fixminer-core/python/data/gumInputLinux
#redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
#redisPath = /Users/anil.koyuncu/projects/richedit/richedit-core/python/data/redis
redisPath = /Users/anilkoyuncu/projects/fixminer/fixminer-core/python/data/redis
#srcMLPath= /Users/anil.koyuncu/Downloads/srcML/src2srcml
srcMLPath= /usr/local/bin/srcml
@@ -1,7 +1,7 @@
package edu.lu.uni.serval.FixPatternMiner;
import edu.lu.uni.serval.fixminer.ediff.EDiffHunkParser;
import edu.lu.uni.serval.fixminer.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.richedit.ediff.EDiffHunkParser;
import edu.lu.uni.serval.richedit.ediff.HierarchicalActionSet;
import java.io.File;
import java.io.FileInputStream;
@@ -33,7 +33,7 @@ public class BaseTest {
List<HierarchicalActionSet> hierarchicalActionSets = parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath,false);
return hierarchicalActionSets;
}catch (NullPointerException n){
// String cmd = "cp /Users/anil.koyuncu/projects/test/fixminer-data/patches/codeflaws/"+n.getMessage().split(root)[1] + " /Users/anil.koyuncu/projects/test/fixminerC/"+n.getMessage();
// String cmd = "cp /Users/anil.koyuncu/projects/test/richedit-data/patches/codeflaws/"+n.getMessage().split(root)[1] + " /Users/anil.koyuncu/projects/test/fixminerC/"+n.getMessage();
// CallShell cs = new CallShell();
// try {
// cs.runShell(cmd);
@@ -1,8 +1,8 @@
package edu.lu.uni.serval.FixPatternMiner;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.fixminer.ediff.EDiffHunkParser;
import edu.lu.uni.serval.fixminer.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.richedit.ediff.EDiffHunkParser;
import edu.lu.uni.serval.richedit.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.utils.ClusterToPattern;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder;
@@ -16,16 +16,17 @@ import java.io.File;
import java.io.IOException;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
import java.util.ArrayList;
@Ignore
public class HunkParserTest extends BaseTest{
@Test
public void testSimple() throws IOException {
// String input = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/revFiles/7f52f3_3845d29_drivers#pci#host#pcie-altera.c";
// String input = "/Users/anil.koyuncu/projects/test/richedit-core/python/data/gumInputLinux/revFiles/7f52f3_3845d29_drivers#pci#host#pcie-altera.c";
// String root = "//Users/anilkoyuncu/projects/gumInputLinux/";
String root = "/Users/anilkoyuncu/projects/fixminer/fixminer-core/python/data/gumInputLinux/";
String root = "/Users/anilkoyuncu/projects/richedit/richedit-core/python/data/gumInputLinux/";
String filename ="";
// filename ="freebsd_ceca9b8_b864ac4_sys#kern#sched_ule.c"; //too long
// filename ="openbsd_e592ec_39c81a_sys#arch#i386#pci#pci_machdep.c"; //not parseable 56 "parameter_list" "" () ( (57 "parameter" "" () ( (22 "decl" "" () ())))
@@ -91,7 +92,7 @@ public class HunkParserTest extends BaseTest{
public void dumpFnction() throws Exception {
String pattern = "function/20/gstreamer_0af74c_e8bae0_libs#gst#net#gstptpclock.c.txt_0";
// String pattern = "function/20/FFmpeg_4c9d1c_3da860_libavutil#file_open.c.txt_0";
ClusterToPattern.main("6399","/Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis","ALLdumps-gumInput.rdb ",pattern);
ClusterToPattern.main("6399","/Users/anil.koyuncu/projects/richedit/richedit-core/python/data/redis","ALLdumps-gumInput.rdb ",pattern);
}
@Test
@@ -112,7 +113,7 @@ public class HunkParserTest extends BaseTest{
public void testSimpleJava() throws IOException {
String root = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInput/spring-amqp/";
String root = "/Users/anil.koyuncu/projects/test/richedit-core/python/data/gumInput/spring-amqp/";
String filename = "5d6e02_e597c5_spring-rabbit#src#main#java#org#springframework#amqp#rabbit#connection#ConnectionFactoryUtils.java";
File revFile = new File(root +"/revFiles/"+filename);
@@ -134,7 +135,7 @@ public class HunkParserTest extends BaseTest{
final JedisPool outerPool = new JedisPool(PoolBuilder.getPoolConfig(), "localhost",Integer.valueOf("6399"),20000000);
EDiffHunkParser parser = new EDiffHunkParser();
String root = "/Users/anil.koyuncu/projects/fixminer/gumInputLinux/linux/";
String root = "/Users/anil.koyuncu/projects/richedit/gumInputLinux/linux/";
String filename = "43f8987_f596c8_drivers#acpi#nfit#core.c";
File revFile = new File(root + "revFiles/"+ filename);
File prevFile =new File(root + "prevFiles/prev_"+filename);
@@ -142,7 +143,7 @@ public class HunkParserTest extends BaseTest{
String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML/src2srcml";
parser.parseFixPatterns(prevFile,revFile,diffFile,"gumInputLinux",outerPool,srcMLPath,"if",false);
String key = "if/3/linux_bb67dd_0922c7_sound#soc#sof#intel#hda.c.txt_0";
File file2load = new File("/Users/anil.koyuncu/projects/fixminer/dumps/"+ key);
File file2load = new File("/Users/anil.koyuncu/projects/richedit/dumps/"+ key);
byte[] dump = FileUtils.readFileToByteArray(file2load);
// String line = FileHelper.readFile(file2load);
@@ -1,9 +1,8 @@
package edu.lu.uni.serval.FixPatternMiner;
import edu.lu.uni.serval.fixminer.ediff.EDiffHunkParser;
import edu.lu.uni.serval.fixminer.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.richedit.ediff.EDiffHunkParser;
import edu.lu.uni.serval.richedit.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
@@ -1236,7 +1235,7 @@ public class TestInputCases {
List<HierarchicalActionSet> hierarchicalActionSets = parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath,false);
return hierarchicalActionSets;
}catch (NullPointerException n){
String cmd = "cp /Users/anil.koyuncu/projects/test/fixminer-data/patches/"+project+"/"+n.getMessage().split(root)[1] + " /Users/anil.koyuncu/projects/test/fixminerC/"+n.getMessage();
String cmd = "cp /Users/anil.koyuncu/projects/test/richedit-data/patches/"+project+"/"+n.getMessage().split(root)[1] + " /Users/anil.koyuncu/projects/test/fixminerC/"+n.getMessage();
CallShell cs = new CallShell();
try {
cs.runShell(cmd);
@@ -1,7 +1,7 @@
package edu.lu.uni.serval.FixPatternMiner;
import edu.lu.uni.serval.fixminer.ediff.EDiffHunkParser;
import edu.lu.uni.serval.fixminer.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.richedit.ediff.EDiffHunkParser;
import edu.lu.uni.serval.richedit.ediff.HierarchicalActionSet;
import org.apache.commons.lang3.StringUtils;
import org.junit.Assert;
import org.junit.Ignore;
@@ -1,8 +1,8 @@
package edu.lu.uni.serval.FixPatternMiner;
import edu.lu.uni.serval.fixminer.ediff.EDiffHunkParser;
import edu.lu.uni.serval.fixminer.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.richedit.ediff.EDiffHunkParser;
import edu.lu.uni.serval.richedit.ediff.HierarchicalActionSet;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
@@ -1,8 +1,7 @@
package edu.lu.uni.serval.FixPatternMiner;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.fixminer.ediff.EDiffHunkParser;
import edu.lu.uni.serval.fixminer.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.richedit.ediff.HierarchicalActionSet;
import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import org.junit.Assert;