This commit is contained in:
haoyetian
2019-12-16 17:40:37 +01:00
parent d0a632bfb0
commit 571d96eb82
7 changed files with 400 additions and 20 deletions
+87
View File
@@ -0,0 +1,87 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: org.javatuples:javatuples:1.2" level="project" />
<orderEntry type="library" name="Maven: com.typesafe.akka:akka-actor_2.11:2.4.11" level="project" />
<orderEntry type="library" name="Maven: org.scala-lang:scala-library:2.11.8" level="project" />
<orderEntry type="library" name="Maven: com.typesafe:config:1.3.0" level="project" />
<orderEntry type="library" name="Maven: org.scala-lang.modules:scala-java8-compat_2.11:0.7.0" level="project" />
<orderEntry type="module" module-name="core" />
<orderEntry type="library" name="Maven: com.github.mpkorstanje:simmetrics-core:3.0.3" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:18.0" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.10" level="project" />
<orderEntry type="library" name="Maven: net.sf.trove4j:trove4j:3.0.3" level="project" />
<orderEntry type="library" name="Maven: com.google.code.gson:gson:2.3" level="project" />
<orderEntry type="module" module-name="gen.jdt" />
<orderEntry type="module" module-name="simple-utils" />
<orderEntry type="library" name="Maven: org.apache.poi:poi:3.12" level="project" />
<orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml:3.12" level="project" />
<orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml-schemas:3.12" level="project" />
<orderEntry type="library" name="Maven: org.apache.xmlbeans:xmlbeans:2.6.0" level="project" />
<orderEntry type="library" name="Maven: stax:stax-api:1.0.1" level="project" />
<orderEntry type="library" name="Maven: net.sourceforge.jexcelapi:jxl:2.6.12" level="project" />
<orderEntry type="library" name="Maven: log4j:log4j:1.2.14" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.core:runtime:3.10.0-v20140318-2214" level="project" />
<orderEntry type="library" name="Maven: org.eclipse:osgi:3.10.0-v20140606-1445" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.equinox:common:3.6.200-v20130402-1505" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.core:jobs:3.6.0-v20140424-0053" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.equinox:registry:3.5.400-v20140428-1507" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.equinox:preferences:3.5.200-v20140224-1527" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.core:contenttype:3.4.200-v20140207-1251" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.equinox:app:1.3.200-v20130910-1609" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.birt.runtime:org.eclipse.core.resources:3.10.0.v20150423-0755" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.tycho:org.eclipse.jdt.core:3.12.2.v20161117-1814" level="project" />
<orderEntry type="module" module-name="gen.srcml" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.7" level="project" />
<orderEntry type="library" name="Maven: ch.qos.logback:logback-classic:1.1.2" level="project" />
<orderEntry type="library" name="Maven: ch.qos.logback:logback-core:1.1.2" level="project" />
<orderEntry type="library" name="Maven: junit:junit:4.11" level="project" />
<orderEntry type="library" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
<orderEntry type="library" name="Maven: redis.clients:jedis:2.8.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-pool2:2.4.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-text:1.3" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.7" level="project" />
<orderEntry type="library" name="Maven: com.rabbitmq:amqp-client:4.0.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-core:4.8.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-analyzers-common:4.8.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.6.1" level="project" />
<orderEntry type="library" name="Maven: com.thoughtworks.xstream:xstream:1.3.1" level="project" />
<orderEntry type="library" name="Maven: xpp3:xpp3_min:1.1.4c" level="project" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />
<orderEntry type="library" name="Maven: org.redisson:redisson:3.3.0" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-common:4.1.8.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec:4.1.8.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-buffer:4.1.8.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport:4.1.8.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-resolver:4.1.8.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-handler:4.1.8.Final" level="project" />
<orderEntry type="library" name="Maven: javax.cache:cache-api:1.0.0" level="project" />
<orderEntry type="library" name="Maven: io.projectreactor:reactor-stream:2.0.8.RELEASE" level="project" />
<orderEntry type="library" name="Maven: io.projectreactor:reactor-core:2.0.8.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.reactivestreams:reactive-streams:1.0.0" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.7.6" level="project" />
<orderEntry type="library" name="Maven: org.yaml:snakeyaml:1.15" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.7.6" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.7.6" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.7.0" level="project" />
<orderEntry type="library" name="Maven: net.openhft:zero-allocation-hashing:0.5" level="project" />
<orderEntry type="library" name="Maven: net.bytebuddy:byte-buddy:1.4.26" level="project" />
<orderEntry type="library" name="Maven: org.jodd:jodd-bean:3.7.1" level="project" />
<orderEntry type="library" name="Maven: org.jodd:jodd-core:3.7.1" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.jetty:jetty-util:9.4.4.v20170414" level="project" />
<orderEntry type="library" name="Maven: com.esotericsoftware:kryo:4.0.2" level="project" />
<orderEntry type="library" name="Maven: com.esotericsoftware:reflectasm:1.11.3" level="project" />
<orderEntry type="library" name="Maven: org.ow2.asm:asm:5.0.4" level="project" />
<orderEntry type="library" name="Maven: com.esotericsoftware:minlog:1.3.0" level="project" />
<orderEntry type="library" name="Maven: org.objenesis:objenesis:2.5.1" level="project" />
</component>
</module>
+7
View File
@@ -50,6 +50,12 @@
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.github.gumtreediff</groupId>
<artifactId>gen.srcml</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<!-- SLF4J - API -->
<dependency>
<groupId>org.slf4j</groupId>
@@ -157,6 +163,7 @@
</dependency>
</dependencies>
<build>
@@ -22,8 +22,8 @@ public class Launcher {
Properties appProps = new Properties();
// String appConfigPath = "/Users/anilkoyuncu/bugStudy/release/code/app.properties";
String appConfigPath = args[0];
String appConfigPath = "/Users/haoyetian/Documents/Lu_code/FixMiner/fixminer_source/src/main/resource/app.properties";
// String appConfigPath = args[0];
appProps.load(new FileInputStream(appConfigPath));
String portInner = appProps.getProperty("portInner","6380");
@@ -36,8 +36,10 @@ public class Launcher {
String input = appProps.getProperty("inputPath","FORKJOIN");
String redisPath = appProps.getProperty("redisPath","FORKJOIN");
String parameter = args[2];
String jobType = args[1];
// String parameter = args[2];
String parameter = null;
// String jobType = args[1];
String jobType = "RICHEDITSCRIPT";
// String parameters = String.format("\nportInner %s " +
// "\nnumOfWorkers %s " +
@@ -1,7 +1,9 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.gen.srcml.GumTreeCComparer;
import edu.lu.uni.serval.gumtree.GumTreeComparer;
import edu.lu.uni.serval.utils.ListSorter;
import redis.clients.jedis.JedisPool;
@@ -39,7 +41,16 @@ public class EDiffParser extends Parser {
protected List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree2(File prevFile, File revFile) {
List<HierarchicalActionSet> actionSets = new ArrayList<>();
// GumTree results
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
boolean isJava =false;
List<Action> gumTreeResults = null;
if (revFile.getName().endsWith(".c") & prevFile.getName().endsWith(".c")){
// gumTreeResults = new GumTreeComparer().compareCFilesWithGumTree(prevFile, revFile);
gumTreeResults = new GumTreeCComparer().compareCFilesWithGumTree(prevFile, revFile);
}else{
gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
isJava = true;
}
if (gumTreeResults == null) {
this.resultType = 1;
return null;
@@ -48,16 +59,21 @@ public class EDiffParser extends Parser {
return actionSets;
} else {
// Regroup GumTre results.
List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
List<HierarchicalActionSet> allActionSets = null;
if (isJava){
allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
}else{
allActionSets = new HierarchicalRegrouperForC().regroupGumTreeResults(gumTreeResults);
}
ListSorter<HierarchicalActionSet> sorter = new ListSorter<>(allActionSets);
actionSets = sorter.sortAscending();
if (actionSets.size() == 0) {
this.resultType = 3;
}
return actionSets;
}
}
@@ -0,0 +1,268 @@
package edu.lu.uni.serval.fixminer.akka.ediff;
import com.github.gumtreediff.actions.model.Action;
import com.github.gumtreediff.actions.model.*;
import com.github.gumtreediff.io.CNodeMap;
import com.github.gumtreediff.tree.ITree;
import edu.lu.uni.serval.gumtree.GumTreeComparer;
import edu.lu.uni.serval.utils.ListSorter;
import java.util.ArrayList;
import java.util.List;
/**
* Regroup GumTree results to a hierarchical construction.
*
* @author kui.liu
*
*/
public class HierarchicalRegrouperForC {
// public static void main(String[] args) {
// GumTreeComparer com = new GumTreeComparer();
// File cFile1 = new File("/Users/anilkoyuncu/bugStudy/dataset/GumTreeInput/linux-stable/prevFiles/prev_0a3d00_b404bc_drivers#pci#iov.c");
// File cFile2 = new File("/Users/anilkoyuncu/bugStudy/dataset/GumTreeInput/linux-stable/revFiles/0a3d00_b404bc_drivers#pci#iov.c");
// List<Action> action = com.compareTwoFilesWithGumTreeForCCode(cFile1, cFile2);
// List<HierarchicalActionSet> actionSet = new HierarchicalRegrouperForC().regroupGumTreeResults(action);
// System.out.println(actionSet);
// }
List<HierarchicalActionSet> actionSets = new ArrayList<>();
public List<HierarchicalActionSet> regroupGumTreeResults(List<Action> actions) {
/*
* First, sort actions by their positions.
*/
// List<Action> actions = new ListSorter<Action>(actionsArgu).sortAscending();
// if (actions == null) {
// actions = actionsArgu;
// }
/*
* Second, group actions by their positions.
*/
HierarchicalActionSet actionSet = null;
for(Action act : actions){
Action parentAct = findParentAction(act, actions);
if (parentAct == null) {
actionSet = createActionSet(act, parentAct, null);
actionSets.add(actionSet);
} else {
if (!addToAactionSet(act, parentAct, actionSets)) {
// The index of the parent action in the actions' list is larger than the index of this action.
actionSet = createActionSet(act, parentAct, null);
actionSets.add(actionSet);
}
}
}
/*
* Third, add the subActionSet to its parent ActionSet.
*/
List<HierarchicalActionSet> reActionSets = new ArrayList<>();
for (HierarchicalActionSet actSet : actionSets) {
Action parentAct = actSet.getParentAction();
if (parentAct != null) {
addToActionSets(actSet, parentAct, actionSets);
} else {
// TypeDeclaration, FieldDeclaration, MethodDeclaration, Statement.
// CatchClause, ConstructorInvocation, SuperConstructorInvocation, SwitchCase
// String astNodeType = actSet.getAstNodeType();
// if (astNodeType.endsWith("TypeDeclaration") || astNodeType.endsWith("FieldDeclaration") || astNodeType.endsWith("EnumDeclaration") ||
// astNodeType.endsWith("MethodDeclaration") || astNodeType.endsWith("Statement") ||
// astNodeType.endsWith("ConstructorInvocation") || astNodeType.endsWith("CatchClause") || astNodeType.endsWith("SwitchCase")) {
reActionSets.add(actSet);
// }
}
}
return reActionSets;
}
private HierarchicalActionSet createActionSet(Action act, Action parentAct, HierarchicalActionSet parent) {
HierarchicalActionSet actionSet = new HierarchicalActionSet();
actionSet.setAction(act);
actionSet.setActionString(parseAction(act.toString()));
actionSet.setParentAction(parentAct);
actionSet.setNode(act.getNode());
actionSet.setParent(parent);
return actionSet;
}
private String parseAction(String actStr1) {
// UPD 25@@!a from !a to isTrue(a) at 69
String[] actStrArrays = actStr1.split("@@");
String actStr = "";
int length = actStrArrays.length;
for (int i =0; i < length - 1; i ++) {
String actStrFrag = actStrArrays[i];
int index = actStrFrag.lastIndexOf(" ") + 1;
String nodeType = actStrFrag.substring(index);
if (!"".equals(nodeType)) {
if (Character.isDigit(nodeType.charAt(0)) || (nodeType.startsWith("-") && Character.isDigit(nodeType.charAt(1)))) {
try {
int typeInt = Integer.parseInt(nodeType);
if (CNodeMap.map.containsKey(typeInt)) {
String type = CNodeMap.map.get(Integer.parseInt(nodeType));
nodeType = type;
}
} catch (NumberFormatException e) {
nodeType = actStrFrag.substring(index);
}
}
}
actStrFrag = actStrFrag.substring(0, index) + nodeType + "@@";
actStr += actStrFrag;
}
actStr += actStrArrays[length - 1];
return actStr;
}
private void addToActionSets(HierarchicalActionSet actionSet, Action parentAct, List<HierarchicalActionSet> actionSets) {
Action act = actionSet.getAction();
for (HierarchicalActionSet actSet : actionSets) {
if (actSet.equals(actionSet)) continue;
Action action = actSet.getAction();
if (!areRelatedActions(action, act)) continue;
if (action.equals(parentAct)) { // actSet is the parent of actionSet.
actionSet.setParent(actSet);
actSet.getSubActions().add(actionSet);
sortSubActions(actSet);
break;
} else {
if (isPossibileSubAction(action, act)) {
// SubAction range startPosition2 <= startPosition && startPosition + length <= startPosition2 + length2
addToActionSets(actionSet, parentAct, actSet.getSubActions());
}
}
}
}
private boolean isPossibileSubAction(Action parent, Action child) {
if ((parent instanceof Update && !(child instanceof Addition))
|| (parent instanceof Delete && child instanceof Delete)
|| (parent instanceof Insert && (child instanceof Insert))) {
int startPosition = child.getPosition();
int length = child.getLength();
int startPosition2 = parent.getPosition();
int length2 = parent.getLength();
if (!(startPosition2 <= startPosition && startPosition + length <= startPosition2 + length2)) {
// when act is not the sub-set of action.
return false;
}
}
return true;
}
private void sortSubActions(HierarchicalActionSet actionSet) {
ListSorter<HierarchicalActionSet> sorter = new ListSorter<HierarchicalActionSet>(actionSet.getSubActions());
List<HierarchicalActionSet> subActions = sorter.sortAscending();
if (subActions != null) {
actionSet.setSubActions(subActions);
}
}
private boolean addToAactionSet(Action act, Action parentAct, List<HierarchicalActionSet> actionSets) {
for(HierarchicalActionSet actionSet : actionSets) {
Action action = actionSet.getAction();
if (!areRelatedActions(action, act)) continue;
if (action.equals(parentAct)) { // actionSet is the parent of actSet.
HierarchicalActionSet actSet = createActionSet(act, actionSet.getAction(), actionSet);
actionSet.getSubActions().add(actSet);
sortSubActions(actionSet);
return true;
} else {
if (isPossibileSubAction(action, act)) {
// SubAction range startPosition2 <= startPosition && startPosition + length <= startP + length2
List<HierarchicalActionSet> subActionSets = actionSet.getSubActions();
if (subActionSets.size() > 0) {
boolean added = addToAactionSet(act, parentAct, subActionSets);
if (added) {
return true;
} else {
continue;
}
}
}
}
}
return false;
}
List<Action> newParentActions = new ArrayList<>();
private Action findParentAction(Action action, List<Action> actions) {
ITree parent = action.getNode().getParent();
if (parent == null) return null;
if (action instanceof Addition) {
parent = ((Addition) action).getParent(); // parent in the fixed source code tree
}
for (Action act : actions) {
if (act.getNode().equals(parent)) {
if (areRelatedActions(act, action)) {
return act;
}
}
}
for (Action act : newParentActions) {
if (act.getNode().equals(parent)) {
if (areRelatedActions(act, action)) {
return act;
}
}
}
ITree tree = action.getNode();
Action parentAction = null;
if (!isStatement(tree)) {
parentAction = new Update(parent, action.getNode().getParent());
newParentActions.add(parentAction);
Action higherParentAct = findParentAction(parentAction, actions);
HierarchicalActionSet actionSet = null;
if (higherParentAct == null) {
actionSet = createActionSet(parentAction, higherParentAct, null);
actionSets.add(actionSet);
} else {
if (!addToAactionSet(parentAction, higherParentAct, actionSets)) {
// The index of the parent action in the actions' list is larger than the index of this action.
actionSet = createActionSet(parentAction, higherParentAct, null);
actionSets.add(actionSet);
}
}
}
return parentAction;
}
private boolean isStatement(ITree tree) {
int nodeType = tree.getType();
if (nodeType == 11 || nodeType == 16 || nodeType == 18 || nodeType == 21
|| nodeType == 22 || nodeType == 23 || nodeType == 24 || nodeType == 84
|| 30 == nodeType || nodeType == 31 || nodeType == 32 || nodeType == 33
|| nodeType == 34 || nodeType == 35 || nodeType == 36 || nodeType == 40
|| nodeType == 41 || nodeType == 49 || nodeType == 73) {// TODO
return true;
}
return false;
}
private boolean areRelatedActions(Action parent, Action child) {
if (parent instanceof Move && !(child instanceof Move)) {// If action is MOV, its children must be MOV.
return false;
}
if (parent instanceof Delete && !(child instanceof Delete)) {// If action is DEL, its children must be DEL.
return false;
}
if (parent instanceof Insert && !(child instanceof Addition)) {// If action is INS, its children must be MOV or INS.
return false;
}
return true;
}
}
@@ -118,7 +118,7 @@ public class EnhancedASTDiff {
log.info(revFilesPath.getPath());
File[] revFiles = revFilesPath.listFiles();
if (revFiles!= null ){
// List<File> collect = Arrays.stream(revFiles).filter(x -> x.getName().startsWith("b50867_6e80c3_src#main#java#org#apache#hadoop#hbase#regionserver#HRegion"))
// List<File> collect = Arrays.stream(revFiles).filter(x -> x.getName().startsWith("0a2756_7598f8_components#camel-cxf#src#main#java#org#apache#camel#component#cxf#CxfHeaderFilterStrategy"))
// .collect(Collectors.toList());// project folders
List<MessageFile> msgFiles = new ArrayList<>();
for (File revFile : revFiles) {
+10 -10
View File
@@ -1,14 +1,14 @@
jobType = LEVEL2
pjName = BugsDotJar
jobType = RICHEDITSCRIPT
pjName = gumInput
portInner = 6380
dbNo = 0
port = 6399
serverWait = 10000
numOfWorkers = 100
pythonPath = /Users/anilkoyuncu/bugStudy/code/python
datasetPath = /Users/anilkoyuncu/bugStudy/release/code
actionType =UPD
threshold = 9
portDumps = 6399
parallelism = AKKA
numOfWorkers = 1
#inputPath = /Users/haoyetian/Documents/Lu_code/FixMiner/fixminer-core/python/data/gumInput
inputPath = /Users/haoyetian/Documents/Lu_code/test
redisPath = /Users/haoyetian/Documents/Lu_code/FixMiner/fixminer-core/python/data/redis
actionType =ALL
eDiffTimeout = 900
#ENHANCEDASTDIFF,CACHE,LEVEL1,LEVEL2,LEVEL3