azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new miner

Browse Source
This commit is contained in:
fixminer
2020-02-17 15:58:40 +01:00
parent 7751215257
commit faac1391b7
14 changed files with 837 additions and 625 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/edu/lu/uni/serval/fixminer/Launcher.java
+2 -2
View File
@@ -22,8 +22,8 @@ public class Launcher {
Properties appProps = new Properties(); Properties appProps = new Properties();
// String appConfigPath = "/Users/anil.koyuncu/projects/fixminer/fixminer_source/src/main/resource/app.properties"; String appConfigPath = "/Users/anilkoyuncu/projects/fixminer/fixminer_source/src/main/resource/app.properties";
String appConfigPath = args[0]; // String appConfigPath = args[0];
appProps.load(new FileInputStream(appConfigPath)); appProps.load(new FileInputStream(appConfigPath));
// String portInner = appProps.getProperty("portInner","6380"); // String portInner = appProps.getProperty("portInner","6380");
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/EDiffParser.java
+2 -1
View File
@@ -43,7 +43,8 @@ public class EDiffParser extends Parser {
// GumTree results // GumTree results
boolean isJava =false; boolean isJava =false;
List<Action> gumTreeResults = null; List<Action> gumTreeResults = null;
if (revFile.getName().endsWith(".c") & prevFile.getName().endsWith(".c") || revFile.getName().endsWith(".h") & prevFile.getName().endsWith(".h")){ if (true){
// if (revFile.getName().endsWith(".c") & prevFile.getName().endsWith(".c") || revFile.getName().endsWith(".h") & prevFile.getName().endsWith(".h")){
// gumTreeResults = new GumTreeComparer().compareCFilesWithGumTree(prevFile, revFile); // gumTreeResults = new GumTreeComparer().compareCFilesWithGumTree(prevFile, revFile);
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HierarchicalRegrouperForC.java
+52 -22
View File
@@ -88,6 +88,7 @@ public class HierarchicalRegrouperForC {
for(HierarchicalActionSet a:reActionSets){ for(HierarchicalActionSet a:reActionSets){
HierarchicalActionSet hierarchicalActionSet = removeBlocks(a); HierarchicalActionSet hierarchicalActionSet = removeBlocks(a);
hierarchicalActionSet = removeIFthenBlocks(hierarchicalActionSet); hierarchicalActionSet = removeIFthenBlocks(hierarchicalActionSet);
hierarchicalActionSet = removeParentForSingle(hierarchicalActionSet);
reActionSets1.add(hierarchicalActionSet); reActionSets1.add(hierarchicalActionSet);
} }
@@ -97,19 +98,23 @@ public class HierarchicalRegrouperForC {
private HierarchicalActionSet removeBlocks(HierarchicalActionSet actionSet){ private HierarchicalActionSet removeBlocks(HierarchicalActionSet actionSet){
List<HierarchicalActionSet> subActions = actionSet.getSubActions(); List<HierarchicalActionSet> subActions = actionSet.getSubActions();
Action action = actionSet.getAction();
if (subActions.size() == 1){ if (subActions.size() == 1){
HierarchicalActionSet subaction = subActions.get(0); HierarchicalActionSet subaction = subActions.get(0);
Action action1 = subaction.getAction();
//else,then,block //else,then,block
if(subaction.getAstNodeType().equals("block")){//|| subaction.getAstNodeType().equals("then") || subaction.getAstNodeType().equals("else")){ if(action.getClass().equals(action1.getClass())) {
List<HierarchicalActionSet> subSubActions = subaction.getSubActions(); if (subaction.getAstNodeType().equals("block")) {//|| subaction.getAstNodeType().equals("then") || subaction.getAstNodeType().equals("else")){
if(subSubActions.size() == 1){ List<HierarchicalActionSet> subSubActions = subaction.getSubActions();
if (subSubActions.size() == 1) {
HierarchicalActionSet subsubsubAction = subSubActions.get(0); HierarchicalActionSet subsubsubAction = subSubActions.get(0);
List<Integer> keysByValue = NodeMap_new.getKeysByValue(NodeMap_new.StatementMap, subsubsubAction.getAstNodeType()); List<Integer> keysByValue = NodeMap_new.getKeysByValue(NodeMap_new.StatementMap, subsubsubAction.getAstNodeType());
if(keysByValue != null && keysByValue.size() ==1){ if (keysByValue != null && keysByValue.size() == 1) {
subsubsubAction.setParent(null); subsubsubAction.setParent(null);
return removeBlocks(subsubsubAction); return removeBlocks(subsubsubAction);
}
} }
} }
} }
@@ -117,29 +122,54 @@ public class HierarchicalRegrouperForC {
return actionSet; return actionSet;
} }
private HierarchicalActionSet removeIFthenBlocks(HierarchicalActionSet actionSet){
private HierarchicalActionSet removeParentForSingle(HierarchicalActionSet actionSet){
List<HierarchicalActionSet> subActions = actionSet.getSubActions(); List<HierarchicalActionSet> subActions = actionSet.getSubActions();
Action action = actionSet.getAction();
if (subActions.size() == 1){ if (subActions.size() == 1){
HierarchicalActionSet subaction = subActions.get(0); HierarchicalActionSet subaction = subActions.get(0);
//else,then,block //else,then,block
if(subaction.getAstNodeType().equals("then")|| subaction.getAstNodeType().equals("else")){//|| subaction.getAstNodeType().equals("then") || subaction.getAstNodeType().equals("else")){ Action action1 = subaction.getAction();
List<HierarchicalActionSet> subSubActions = subaction.getSubActions(); //else,then,block
if(subSubActions.size() == 1){ if(action.getClass().equals(action1.getClass())) {
if(!(subaction.getAstNodeType().equals("condition") || subaction.getAstNodeType().equals("init"))){
List<Integer> keysByValue = NodeMap_new.getKeysByValue(NodeMap_new.StatementMap, subaction.getAstNodeType());
if(keysByValue != null && keysByValue.size() ==1){
subaction.setParent(null);
return removeParentForSingle(subaction);
}}}
}
return actionSet;
HierarchicalActionSet subsubsubAction = subSubActions.get(0); }
if(subsubsubAction.getAstNodeType().equals("block")){ private HierarchicalActionSet removeIFthenBlocks(HierarchicalActionSet actionSet){
List<HierarchicalActionSet> subActions1 = subsubsubAction.getSubActions(); List<HierarchicalActionSet> subActions = actionSet.getSubActions();
if (subActions1.size() ==1){ Action action = actionSet.getAction();
HierarchicalActionSet hierarchicalActionSet = subActions1.get(0); if (subActions.size() == 1){
List<Integer> keysByValue = NodeMap_new.getKeysByValue(NodeMap_new.StatementMap, hierarchicalActionSet.getAstNodeType()); HierarchicalActionSet subaction = subActions.get(0);
if(keysByValue != null && keysByValue.size() ==1){ //else,then,block
hierarchicalActionSet.setParent(null); Action action1 = subaction.getAction();
return removeBlocks(hierarchicalActionSet); //else,then,block
if(action.getClass().equals(action1.getClass())) {
if (subaction.getAstNodeType().equals("then") || subaction.getAstNodeType().equals("else")) {//|| subaction.getAstNodeType().equals("then") || subaction.getAstNodeType().equals("else")){
List<HierarchicalActionSet> subSubActions = subaction.getSubActions();
if (subSubActions.size() == 1) {
HierarchicalActionSet subsubsubAction = subSubActions.get(0);
if (subsubsubAction.getAstNodeType().equals("block")) {
List<HierarchicalActionSet> subActions1 = subsubsubAction.getSubActions();
if (subActions1.size() == 1) {
HierarchicalActionSet hierarchicalActionSet = subActions1.get(0);
List<Integer> keysByValue = NodeMap_new.getKeysByValue(NodeMap_new.StatementMap, hierarchicalActionSet.getAstNodeType());
if (keysByValue != null && keysByValue.size() == 1) {
hierarchicalActionSet.setParent(null);
return removeBlocks(hierarchicalActionSet);
}
} }
} }
}
}
} }
} }
} }
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/HunkParserTest.java
+13 -11
View File
@@ -10,6 +10,7 @@ import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.PoolBuilder; import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.javatuples.Pair; import org.javatuples.Pair;
import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
@@ -31,8 +32,8 @@ public class HunkParserTest {
public void testSimple() throws IOException { public void testSimple() throws IOException {
// String input = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/revFiles/7f52f3_3845d29_drivers#pci#host#pcie-altera.c"; // String input = "/Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/revFiles/7f52f3_3845d29_drivers#pci#host#pcie-altera.c";
// String root = "/Users/anil.koyuncu/projects/fixminer/gumInputLinux/linux/"; // String root = "//Users/anilkoyuncu/projects/gumInputLinux/";
String root = "/Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/gumInputLinux/"; String root = "/Users/anilkoyuncu/projects/fixminer/fixminer-core/python/data/gumInputLinux/";
String filename =""; String filename ="";
// filename ="freebsd_ceca9b8_b864ac4_sys#kern#sched_ule.c"; //too long // filename ="freebsd_ceca9b8_b864ac4_sys#kern#sched_ule.c"; //too long
// filename ="openbsd_e592ec_39c81a_sys#arch#i386#pci#pci_machdep.c"; //not parseable 56 "parameter_list" "" () ( (57 "parameter" "" () ( (22 "decl" "" () ()))) // filename ="openbsd_e592ec_39c81a_sys#arch#i386#pci#pci_machdep.c"; //not parseable 56 "parameter_list" "" () ( (57 "parameter" "" () ( (22 "decl" "" () ())))
@@ -52,21 +53,21 @@ public class HunkParserTest {
// filename ="omp_19fae3_1e4dcd_src#mca#mpool#sm#mpool_sm_mmap.c"; // cannot find // filename ="omp_19fae3_1e4dcd_src#mca#mpool#sm#mpool_sm_mmap.c"; // cannot find
// filename ="FFmpeg_a8343bf_2b2039_libavformat#riff.c"; // ok // filename ="FFmpeg_a8343bf_2b2039_libavformat#riff.c"; // ok
// filename ="freebsd_32766e4_200ff4_sbin#routed#parms.c"; // ok // filename ="freebsd_32766e4_200ff4_sbin#routed#parms.c"; // ok
filename ="openbsd_150ddd_cf0e20_usr.sbin#user#user.c"; //notok // filename ="openbsd_150ddd_cf0e20_usr.sbin#user#user.c"; //notok
filename ="openbsd_6fac1e_c3b383_usr.bin#tmux#window-copy.c"; //notok // filename ="openbsd_6fac1e_c3b383_usr.bin#tmux#window-copy.c"; //notok
filename ="freebsd_0cb6f2_b4c742_sys#dev#ipw#if_ipw.c"; //notok // filename ="freebsd_0cb6f2_b4c742_sys#dev#ipw#if_ipw.c"; //notok
filename ="php-src_7defd5_da06f7_ext#mbstring#mbstring.c"; //notok (19 "expr_stmt" "" () ())))) // filename ="php-src_7defd5_da06f7_ext#mbstring#mbstring.c"; //notok (19 "expr_stmt" "" () ()))))
filename ="libtiff_177169_71715f_tools#tiff2ps.c"; //notok (19 "expr_stmt" "" () ())))) // filename ="libtiff_177169_71715f_tools#tiff2ps.c"; //notok (19 "expr_stmt" "" () ()))))
filename ="linux_955c1dd_0aaee4_drivers#gpu#drm#i915#gvt#handlers.c"; //notok (19 "expr_stmt" "" () ())))) filename ="linux_955c1dd_0aaee4_drivers#gpu#drm#i915#gvt#handlers.c"; //notok (19 "expr_stmt" "" () ()))))
// filename ="FFmpeg_0726b2_66d2ff_libav#jpeg.c"; // filename ="FFmpeg_0726b2_66d2ff_libav#jpeg.c";
String pj = filename.split("_")[0]; String pj = filename.split("_")[0];
filename = filename.replace(pj+"_",""); filename = filename.replace(pj+"_","");
root = root + pj + "/"; // root = root + pj + "/";
// root = root + "codeflaws/"; root = root + "codeflaws/";
filename ="287-A-14208510-14208532.c";
// filename = "474-A-15925943-15925951.c"; //mot ok // filename = "474-A-15925943-15925951.c"; //mot ok
// filename = "6-C-11536006-11536039.c"; //okish // filename = "6-C-11536006-11536039.c"; //okish
// filename = "500-A-18298071-18298124.c"; //ok // filename = "500-A-18298071-18298124.c"; //ok
@@ -85,7 +86,7 @@ public class HunkParserTest {
EDiffHunkParser parser = new EDiffHunkParser(); EDiffHunkParser parser = new EDiffHunkParser();
String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml"; String srcMLPath = "/usr/local/bin/srcml";
// String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml"; // String srcMLPath = "/Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml";
parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath); parser.parseChangedSourceCodeWithGumTree2(prevFile, revFile, srcMLPath);
// ITree t = new SrcmlCppTreeGenerator().generateFromFile(input).getRoot(); // ITree t = new SrcmlCppTreeGenerator().generateFromFile(input).getRoot();
@@ -93,6 +94,7 @@ public class HunkParserTest {
} }
@Test @Test
public void dumpFnction() throws Exception { public void dumpFnction() throws Exception {
String pattern = "function/20/gstreamer_0af74c_e8bae0_libs#gst#net#gstptpclock.c.txt_0"; String pattern = "function/20/gstreamer_0af74c_e8bae0_libs#gst#net#gstptpclock.c.txt_0";
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestIntroClass.java
+446
View File
File diff suppressed because one or more lines are too long
src/main/java/edu/lu/uni/serval/fixminer/akka/ediff/TestRealCases.java
+272 -579
View File
File diff suppressed because it is too large Load Diff
src/main/java/edu/lu/uni/serval/fixminer/jobs/EnhancedASTDiff.java
+6 -6
View File
@@ -4,8 +4,6 @@ import akka.actor.ActorRef;
import akka.actor.ActorSystem; import akka.actor.ActorSystem;
import edu.lu.uni.serval.fixminer.akka.ediff.*; import edu.lu.uni.serval.fixminer.akka.ediff.*;
import edu.lu.uni.serval.utils.CallShell; import edu.lu.uni.serval.utils.CallShell;
import edu.lu.uni.serval.utils.EDiffHelper;
import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.utils.PoolBuilder; import edu.lu.uni.serval.utils.PoolBuilder;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
@@ -13,9 +11,9 @@ import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
import java.io.*; import java.io.File;
import java.nio.file.Files; import java.io.FileNotFoundException;
import java.nio.file.Paths; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
@@ -64,6 +62,7 @@ public class EnhancedASTDiff {
.filter(x -> !x.getName().startsWith(".")) .filter(x -> !x.getName().startsWith("."))
.filter(x -> !x.getName().startsWith("cocci")) .filter(x -> !x.getName().startsWith("cocci"))
.filter(x -> !x.getName().endsWith(".index")) .filter(x -> !x.getName().endsWith(".index"))
.filter(x -> x.getName().endsWith("codeflaws"))
.collect(Collectors.toList()); .collect(Collectors.toList());
@@ -152,7 +151,8 @@ public class EnhancedASTDiff {
File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
fileName = fileName + ".txt"; fileName = fileName + ".txt";
File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file
// if(FileHelper.readFile(diffentryFile).split("@@\\s\\-\\d+,*\\d*\\s\\+\\d+,*\\d*\\s@@").length > 2)
// continue;
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
msgFiles.add(msgFile); msgFiles.add(msgFile);
src/main/resource/app.properties
+5 -4
View File
@@ -3,15 +3,16 @@ pjName = gumInput
portInner = 6380 portInner = 6380
portDumps = 6399 portDumps = 6399
parallelism = AKKA parallelism = AKKA
numOfWorkers = 30 numOfWorkers = 14
hostname = localhost hostname = localhost
hunkLimit = 10 hunkLimit = 10
#inputPath = /Users/anilkoyuncu/projects/gumInputLinux #inputPath = /Users/anilkoyuncu/projects/gumInputLinux
inputPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/gumInputLinux inputPath = /Users/anilkoyuncu/projects/fixminer/fixminer-core/python/data/gumInputLinux
redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis #redisPath = /Users/anil.koyuncu/projects/fixminer/fixminer-core/python/data/redis
redisPath = /Users/anilkoyuncu/projects/fixminer/fixminer-core/python/data/redis
#srcMLPath= /Users/anil.koyuncu/Downloads/srcML/src2srcml #srcMLPath= /Users/anil.koyuncu/Downloads/srcML/src2srcml
srcMLPath= /Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml srcMLPath= /usr/local/bin/srcml
actionType = ALL actionType = ALL
eDiffTimeout = 900 eDiffTimeout = 900
src/main/resource/testFiles/if_else.c
+10
View File
@@ -0,0 +1,10 @@
const static ctl_serialize_action
static int test(){
field = ATOM(TST);
if(IS_ERR(fields->mode))
return PTR_ERROR(fields->mode);
if(a>0)
return 1;
return 0;
}
src/main/resource/testFiles/if_return.c
+6
View File
@@ -0,0 +1,6 @@
static int test(){
field = ATOM(TST);
if(IS_ERR(fields->mode))
return PTR_ERROR(fields->mode);
return 0;
}
src/main/resource/testFiles/prev_if_else.c
+9
View File
@@ -0,0 +1,9 @@
static ctl_serialize_action
static int test(){
field = ATOM(TST);
if(IS_ERR(fields->mode))
return PTR_ERROR(fields->mode);
else if(a>0)
return 1;
return 0;
}
src/main/resource/testFiles/prev_if_return.c
+4
View File
@@ -0,0 +1,4 @@
static int test(){
field = ATOM(TST);
return PTR_ERROR_OR_ZERO(fields->mode);
}
src/main/resource/testFiles/prev_struct.c
+5
View File
@@ -0,0 +1,5 @@
struct b {
int x;
double y;
float z;
};
src/main/resource/testFiles/struct.c
+5
View File
@@ -0,0 +1,5 @@
struct a {
int x;
double y;
float z;
};