commit for c parser
This commit is contained in:
Anil Koyuncu
@@ -0,0 +1,59 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
|
||||
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8" inherit-compiler-output="false">
|
||||
<output url="file://$MODULE_DIR$/target/classes" />
|
||||
<output-test url="file://$MODULE_DIR$/target/test-classes" />
|
||||
<content url="file://$MODULE_DIR$">
|
||||
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
|
||||
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/target" />
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
<orderEntry type="module" module-name="simple-utils" />
|
||||
<orderEntry type="library" name="Maven: org.apache.poi:poi:3.12" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.9" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml:3.12" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml-schemas:3.12" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.xmlbeans:xmlbeans:2.6.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: stax:stax-api:1.0.1" level="project" />
|
||||
<orderEntry type="library" name="Maven: net.sourceforge.jexcelapi:jxl:2.6.12" level="project" />
|
||||
<orderEntry type="library" name="Maven: log4j:log4j:1.2.14" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.25" level="project" />
|
||||
<orderEntry type="library" name="Maven: ch.qos.logback:logback-core:1.2.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: ch.qos.logback:logback-classic:1.2.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.typesafe.akka:akka-actor_2.11:2.4.11" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.scala-lang:scala-library:2.11.8" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.typesafe:config:1.3.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.scala-lang.modules:scala-java8-compat_2.11:0.7.0" level="project" />
|
||||
<orderEntry type="module" module-name="core" />
|
||||
<orderEntry type="library" name="Maven: com.github.mpkorstanje:simmetrics-core:3.0.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.google.guava:guava:18.0" level="project" />
|
||||
<orderEntry type="library" name="Maven: net.sf.trove4j:trove4j:3.0.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.google.code.gson:gson:2.3" level="project" />
|
||||
<orderEntry type="module" module-name="gen.jdt" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.core:runtime:3.10.0-v20140318-2214" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse:osgi:3.10.0-v20140606-1445" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.equinox:common:3.6.200-v20130402-1505" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.core:jobs:3.6.0-v20140424-0053" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.equinox:registry:3.5.400-v20140428-1507" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.equinox:preferences:3.5.200-v20140224-1527" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.core:contenttype:3.4.200-v20140207-1251" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.equinox:app:1.3.200-v20130910-1609" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.birt.runtime:org.eclipse.core.resources:3.10.0.v20150423-0755" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.tycho:org.eclipse.jdt.core:3.12.2.v20161117-1814" level="project" />
|
||||
<orderEntry type="library" name="Maven: edu.lu.uni.serval:SourceCodeParser:0.0.1-SNAPSHOT" level="project" />
|
||||
<orderEntry type="module" module-name="GitTraveller" />
|
||||
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.7" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.jsoup:jsoup:1.11.2" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.eclipse.jgit:org.eclipse.jgit:4.7.0.201704051617-r" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.jcraft:jsch:0.1.54" level="project" />
|
||||
<orderEntry type="library" name="Maven: com.googlecode.javaewah:JavaEWAH:1.1.6" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.3.6" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.3.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.3" level="project" />
|
||||
<orderEntry type="library" name="Maven: org.javatuples:javatuples:1.2" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
|
||||
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
|
||||
</component>
|
||||
</module>
|
||||
@@ -103,7 +103,7 @@ public class ASTNodeMap {
|
||||
map.put(88, "NameQualifiedType");
|
||||
map.put(89, "CreationReference");
|
||||
map.put(90, "ExpressionMethodReference");
|
||||
map.put(91, "SuperMethhodReference");
|
||||
map.put(91, "SuperMethodReference");
|
||||
map.put(92, "TypeMethodReference");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package edu.lu.uni.serval.FixPattern.utils;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class CNodeMap {
|
||||
|
||||
public static Map<Integer, String> map;
|
||||
|
||||
static {
|
||||
map = new HashMap<Integer, String>();
|
||||
map.put(300100, "If");
|
||||
map.put(410100, "DefineFunc");
|
||||
map.put(340000, "Storage");
|
||||
map.put(240800, "Postfix");
|
||||
map.put(280100, "Goto");
|
||||
map.put(280003, "Return");
|
||||
map.put(300100, "If");
|
||||
map.put(270100, "Label");
|
||||
map.put(290001, "None");
|
||||
map.put(450200, "Definition");
|
||||
map.put(200000, "ParamList");
|
||||
map.put(310300, "For");
|
||||
map.put(241200, "ArrayAccess");
|
||||
map.put(240600, "Sequence");
|
||||
map.put(241600, "SizeOfType");
|
||||
map.put(240500, "CondExpr");
|
||||
map.put(240100, "Ident");
|
||||
map.put(290100, "Some");
|
||||
map.put(280002, "Break");
|
||||
map.put(360100, "InitExpr");
|
||||
map.put(241300, "RecordAccess");
|
||||
map.put(280001, "Continue");
|
||||
map.put(470000, "GenericList");
|
||||
map.put(241500, "SizeOfExpr");
|
||||
map.put(310100, "While");
|
||||
map.put(240900, "Infix");
|
||||
map.put(460000, "Program");
|
||||
map.put(360200, "InitList");
|
||||
map.put(450300, "CppTop");
|
||||
map.put(380000, "Definition");
|
||||
map.put(330000, "Compound");
|
||||
map.put(240700, "Assignment");
|
||||
map.put(241000, "Unary");
|
||||
map.put(280200, "ReturnExpr");
|
||||
map.put(270400, "Default");
|
||||
map.put(420100, "DefineExpr");
|
||||
map.put(241400, "RecordPtAccess");
|
||||
map.put(450800, "FinalDef");
|
||||
map.put(240400, "FunCall");
|
||||
map.put(241100, "Binary");
|
||||
map.put(20100, "Left");
|
||||
map.put(490100, "IfToken");
|
||||
map.put(300200, "Switch");
|
||||
map.put(400200, "Include");
|
||||
map.put(242000, "ParenExpr");
|
||||
map.put(410001, "DefineVar");
|
||||
map.put(310200, "DoWhile");
|
||||
map.put(420001, "DefineEmpty");
|
||||
map.put(241700, "Cast");
|
||||
map.put(270200, "Case");
|
||||
map.put(240200, "Constant");
|
||||
map.put(450100, "Declaration");
|
||||
map.put(480000, "GenericString");
|
||||
map.put(400100, "Define");
|
||||
map.put(450600, "EmptyDef");
|
||||
map.put(220100, "ParameterType");
|
||||
map.put(260300, "ExprStatement");
|
||||
map.put(350100, "DeclList");
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,7 @@ public abstract class Parser implements ParserInterface {
|
||||
protected List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree(File prevFile, File revFile) {
|
||||
List<HierarchicalActionSet> actionSets = new ArrayList<>();
|
||||
// GumTree results
|
||||
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
|
||||
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTreeForCCode(prevFile, revFile);
|
||||
if (gumTreeResults != null && gumTreeResults.size() > 0) {
|
||||
// Regroup GumTre results.
|
||||
List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
|
||||
|
||||
+6
-5
@@ -137,8 +137,9 @@ public class FixedViolationHunkParser extends FixedViolationParser {
|
||||
}
|
||||
|
||||
String patchPosition = "\n" + revFile.getName() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n";
|
||||
// String info = Configuration.PATCH_SIGNAL + "\n" + patchPosition + patchHunk.getHunk() + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets, bugEndPosition, fixEndPosition) + "\n";
|
||||
String info = Configuration.PATCH_SIGNAL + "\n" + patchPosition + patchHunk.getHunk() + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets) + "\n";
|
||||
String info = Configuration.PATCH_SIGNAL + "\n" + patchPosition + patchHunk.getHunk() + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets, bugEndPosition, fixEndPosition) + "\n";
|
||||
//TODO uncomment the line below for more detailed gumtree input
|
||||
// String info = Configuration.PATCH_SIGNAL + "\n" + patchPosition + patchHunk.getHunk() + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets) + "\n";
|
||||
// if (noUpdate(editScriptTokens)) {
|
||||
// }
|
||||
|
||||
@@ -146,9 +147,9 @@ public class FixedViolationHunkParser extends FixedViolationParser {
|
||||
this.sizes += size + "\n";
|
||||
this.astEditScripts += astEditScripts + "\n";
|
||||
|
||||
SimpleTree simpleTree = getBuggyCodeTree(patchHunk, bugEndPosition, prevFile, bugStartLine, bugEndLine);
|
||||
String tokens = Tokenizer.getTokensDeepFirst(simpleTree).trim();
|
||||
this.tokensOfSourceCode += tokens + "\n";
|
||||
// SimpleTree simpleTree = getBuggyCodeTree(patchHunk, bugEndPosition, prevFile, bugStartLine, bugEndLine);
|
||||
// String tokens = Tokenizer.getTokensDeepFirst(simpleTree).trim();
|
||||
// this.tokensOfSourceCode += tokens + "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+4
-3
@@ -12,7 +12,8 @@ import com.github.gumtreediff.actions.model.Action;
|
||||
import edu.lu.uni.serval.FixPatternParser.Parser;
|
||||
import edu.lu.uni.serval.gumtree.GumTreeComparer;
|
||||
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
|
||||
import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper;
|
||||
|
||||
import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouperForC;
|
||||
import edu.lu.uni.serval.utils.FileHelper;
|
||||
import edu.lu.uni.serval.utils.ListSorter;
|
||||
|
||||
@@ -50,7 +51,7 @@ public class FixedViolationParser extends Parser {
|
||||
protected List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree2(File prevFile, File revFile) {
|
||||
List<HierarchicalActionSet> actionSets = new ArrayList<>();
|
||||
// GumTree results
|
||||
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
|
||||
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTreeForCCode(prevFile, revFile);
|
||||
if (gumTreeResults == null) {
|
||||
this.resultType = 1;
|
||||
return null;
|
||||
@@ -59,7 +60,7 @@ public class FixedViolationParser extends Parser {
|
||||
return actionSets;
|
||||
} else {
|
||||
// Regroup GumTre results.
|
||||
List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
|
||||
List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouperForC().regroupGumTreeResults(gumTreeResults);
|
||||
// for (HierarchicalActionSet actionSet : allActionSets) {
|
||||
// String astNodeType = actionSet.getAstNodeType();
|
||||
// if (astNodeType.endsWith("Statement") || "FieldDeclaration".equals(astNodeType)) {
|
||||
|
||||
@@ -2,6 +2,7 @@ package edu.lu.uni.serval.MultipleThreadsParser;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
@@ -31,44 +32,61 @@ public class AkkaParser {
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
public static void main(String[] args) {
|
||||
String inputRootPath = args[0];
|
||||
int numberOfWorkers = Integer.parseInt(args[1]);
|
||||
int hunkThreshold = 0;
|
||||
try {
|
||||
hunkThreshold = Integer.parseInt(args[2]);
|
||||
} catch (NumberFormatException e1) {
|
||||
hunkThreshold = 10;
|
||||
}
|
||||
// String inputRootPath = args[0];
|
||||
int numberOfWorkers = 5; //Integer.parseInt(args[1]);
|
||||
int hunkThreshold = 10;
|
||||
// try {
|
||||
// hunkThreshold = Integer.parseInt(args[2]);
|
||||
// } catch (NumberFormatException e1) {
|
||||
// hunkThreshold = 10;
|
||||
// }
|
||||
|
||||
Configuration.ROOT_PATH = inputRootPath;
|
||||
// Configuration.ROOT_PATH = inputRootPath;
|
||||
Configuration.HUNK_SIZE = hunkThreshold;
|
||||
// input data
|
||||
log.info("Get the input data...");
|
||||
final List<MessageFile> msgFiles = getMessageFiles(Configuration.GUM_TREE_INPUT);
|
||||
log.info("MessageFiles: " + msgFiles.size());
|
||||
|
||||
// output path
|
||||
final String editScriptsFilePath = Configuration.EDITSCRIPTS_FILE_PATH;
|
||||
final String patchesSourceCodeFilePath = Configuration.PATCH_SOURCECODE_FILE_PATH;
|
||||
final String buggyTokensFilePath = Configuration.BUGGY_CODE_TOKEN_FILE_PATH;
|
||||
final String editScriptSizesFilePath = Configuration.EDITSCRIPT_SIZES_FILE_PATH;
|
||||
FileHelper.deleteDirectory(editScriptsFilePath);
|
||||
FileHelper.deleteDirectory(patchesSourceCodeFilePath);
|
||||
FileHelper.deleteDirectory(buggyTokensFilePath);
|
||||
FileHelper.deleteDirectory(editScriptSizesFilePath);
|
||||
|
||||
ActorSystem system = null;
|
||||
ActorRef parsingActor = null;
|
||||
final WorkMessage msg = new WorkMessage(0, msgFiles);
|
||||
try {
|
||||
log.info("Akka begins...");
|
||||
system = ActorSystem.create("Mining-FixPattern-System");
|
||||
parsingActor = system.actorOf(ParseFixPatternActor.props(numberOfWorkers, editScriptsFilePath,
|
||||
patchesSourceCodeFilePath, buggyTokensFilePath, editScriptSizesFilePath), "mine-fix-pattern-actor");
|
||||
parsingActor.tell(msg, ActorRef.noSender());
|
||||
} catch (Exception e) {
|
||||
system.shutdown();
|
||||
e.printStackTrace();
|
||||
List<String> pjList = Arrays.asList("APACHE-CAMEL","APACHE-HBASE","APACHE-HIVE","COMMONS-CODEC","COMMONS-COLLECTIONS",
|
||||
"COMMONS-COMPRESS","COMMONS-CONFIGURATION","COMMONS-CRYPTO","COMMONS-CSV","COMMONS-IO","COMMONS-LANG",
|
||||
"COMMONS-MATH","COMMONS-WEAVER","JBOSS-ENTESB","JBOSS-JBMETA","SPRING-AMQP","SPRING-ANDROID","SPRING-BATCH",
|
||||
"SPRING-BATCHADM","SPRING-DATACMNS","SPRING-DATAGRAPH","SPRING-DATAJPA","SPRING-DATAMONGO","SPRING-DATAREDIS",
|
||||
"SPRING-DATAREST","SPRING-LDAP","SPRING-MOBILE","SPRING-ROO","SPRING-SEC","SPRING-SECOAUTH","SPRING-SGF","SPRING-SHDP",
|
||||
"SPRING-SOCIAL","SPRING-SOCIALFB","SPRING-SOCIALLI","SPRING-SOCIALTW","SPRING-SPR","SPRING-SWF","SPRING-SWS","WILDFLY-ELY",
|
||||
"WILDFLY-SWARM","WILDFLY-WFARQ","WILDFLY-WFCORE","WILDFLY-WFLY","WILDFLY-WFMP");
|
||||
for (String pj : pjList) {
|
||||
String[] split = pj.split("-");
|
||||
String pjPath = split[0];
|
||||
String pjName = split[1];
|
||||
// String rootPath = "/Volumes/data/bugStudy/";
|
||||
// String DATASET_FILE_PATH = rootPath + "/dataset/GumTreeInput/" + pjName;
|
||||
// String GIT_REPOSITORY_PATH = "/Users/anilkoyuncu/bugLocalizationStudy/irblsensitivity/data/" + pjPath + "/" + pjName + "/gitrepo/.git";
|
||||
log.info(pjName);
|
||||
// input data
|
||||
log.info("Get the input data...");
|
||||
final List<MessageFile> msgFiles = getMessageFiles(Configuration.GUM_TREE_INPUT +pjName +"gitrepo/");
|
||||
log.info("MessageFiles: " + msgFiles.size());
|
||||
|
||||
// output path
|
||||
final String editScriptsFilePath = Configuration.EDITSCRIPTS_FILE_PATH + pjName + "/";
|
||||
final String patchesSourceCodeFilePath = Configuration.PATCH_SOURCECODE_FILE_PATH+ pjName + "/";
|
||||
final String buggyTokensFilePath = Configuration.BUGGY_CODE_TOKEN_FILE_PATH+ pjName + "/";
|
||||
final String editScriptSizesFilePath = Configuration.EDITSCRIPT_SIZES_FILE_PATH+ pjName + "/";
|
||||
FileHelper.deleteDirectory(editScriptsFilePath);
|
||||
FileHelper.deleteDirectory(patchesSourceCodeFilePath);
|
||||
FileHelper.deleteDirectory(buggyTokensFilePath);
|
||||
FileHelper.deleteDirectory(editScriptSizesFilePath);
|
||||
|
||||
ActorSystem system = null;
|
||||
ActorRef parsingActor = null;
|
||||
final WorkMessage msg = new WorkMessage(0, msgFiles);
|
||||
try {
|
||||
log.info("Akka begins...");
|
||||
system = ActorSystem.create("Mining-FixPattern-System");
|
||||
parsingActor = system.actorOf(ParseFixPatternActor.props(numberOfWorkers, editScriptsFilePath,
|
||||
patchesSourceCodeFilePath, buggyTokensFilePath, editScriptSizesFilePath), "mine-fix-pattern-actor");
|
||||
parsingActor.tell(msg, ActorRef.noSender());
|
||||
} catch (Exception e) {
|
||||
system.shutdown();
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package edu.lu.uni.serval.MultipleThreadsParser;
|
||||
|
||||
import akka.actor.ActorRef;
|
||||
import akka.actor.ActorSystem;
|
||||
import edu.lu.uni.serval.config.Configuration;
|
||||
import edu.lu.uni.serval.utils.FileHelper;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Multi-thread parser of parsing the difference between buggy code file and fixed code file.
|
||||
*
|
||||
* @author kui.liu
|
||||
*
|
||||
*/
|
||||
public class AkkaParser2 {
|
||||
|
||||
private static Logger log = LoggerFactory.getLogger(AkkaParser2.class);
|
||||
|
||||
/**
|
||||
* Two parameters:
|
||||
* First one: the root path of input data.
|
||||
* Second one: the number of threads.
|
||||
* Third one: the threshold of selecting patch hunks.
|
||||
* @param args
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
public static void main(String[] args) {
|
||||
// String inputRootPath = args[0];
|
||||
int numberOfWorkers = 10; //Integer.parseInt(args[1]);
|
||||
int hunkThreshold = 5;
|
||||
// try {
|
||||
// hunkThreshold = Integer.parseInt(args[2]);
|
||||
// } catch (NumberFormatException e1) {
|
||||
// hunkThreshold = 10;
|
||||
// }
|
||||
|
||||
// Configuration.ROOT_PATH = inputRootPath;
|
||||
Configuration.HUNK_SIZE = hunkThreshold;
|
||||
|
||||
|
||||
|
||||
String pjName = "linux-stable";
|
||||
// String rootPath = "/Volumes/data/bugStudy/";
|
||||
// String DATASET_FILE_PATH = rootPath + "/dataset/GumTreeInput/" + pjName;
|
||||
// String GIT_REPOSITORY_PATH = "/Users/anilkoyuncu/bugLocalizationStudy/irblsensitivity/data/" + pjPath + "/" + pjName + "/gitrepo/.git";
|
||||
log.info(pjName);
|
||||
// input data
|
||||
log.info("Get the input data...");
|
||||
final List<MessageFile> msgFiles = getMessageFiles(Configuration.GUM_TREE_INPUT +pjName +"/");
|
||||
log.info("MessageFiles: " + msgFiles.size());
|
||||
|
||||
// output path
|
||||
final String editScriptsFilePath = Configuration.EDITSCRIPTS_FILE_PATH + pjName + "/";
|
||||
final String patchesSourceCodeFilePath = Configuration.PATCH_SOURCECODE_FILE_PATH+ pjName + "/";
|
||||
final String buggyTokensFilePath = Configuration.BUGGY_CODE_TOKEN_FILE_PATH+ pjName + "/";
|
||||
final String editScriptSizesFilePath = Configuration.EDITSCRIPT_SIZES_FILE_PATH+ pjName + "/";
|
||||
FileHelper.deleteDirectory(editScriptsFilePath);
|
||||
FileHelper.deleteDirectory(patchesSourceCodeFilePath);
|
||||
FileHelper.deleteDirectory(buggyTokensFilePath);
|
||||
FileHelper.deleteDirectory(editScriptSizesFilePath);
|
||||
|
||||
ActorSystem system = null;
|
||||
ActorRef parsingActor = null;
|
||||
final WorkMessage msg = new WorkMessage(0, msgFiles);
|
||||
try {
|
||||
log.info("Akka begins...");
|
||||
system = ActorSystem.create("Mining-FixPattern-System");
|
||||
parsingActor = system.actorOf(ParseFixPatternActor.props(numberOfWorkers, editScriptsFilePath,
|
||||
patchesSourceCodeFilePath, buggyTokensFilePath, editScriptSizesFilePath), "mine-fix-pattern-actor");
|
||||
parsingActor.tell(msg, ActorRef.noSender());
|
||||
} catch (Exception e) {
|
||||
system.shutdown();
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Get violation-related files.
|
||||
*
|
||||
* @param gumTreeInput
|
||||
* @return
|
||||
*/
|
||||
public static List<MessageFile> getMessageFiles(String gumTreeInput) {
|
||||
String inputPath = gumTreeInput; // prevFiles revFiles diffentryFile positionsFile
|
||||
File revFilesPath = new File(inputPath + "revFiles/");
|
||||
File[] revFiles = revFilesPath.listFiles(); // project folders
|
||||
List<MessageFile> msgFiles = new ArrayList<>();
|
||||
|
||||
for (File revFile : revFiles) {
|
||||
if (revFile.getName().endsWith(".c") || revFile.getName().endsWith(".h")) {
|
||||
String fileName = revFile.getName();
|
||||
File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
|
||||
// fileName = fileName.replace(".java", ".txt");
|
||||
File diffentryFile = new File(gumTreeInput + "diffentries/" + fileName); // DiffEntry file
|
||||
File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file
|
||||
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
|
||||
msgFile.setPositionFile(positionFile);
|
||||
msgFiles.add(msgFile);
|
||||
|
||||
}
|
||||
// }
|
||||
}
|
||||
|
||||
return msgFiles.subList(10,20);
|
||||
}
|
||||
}
|
||||
@@ -85,10 +85,10 @@ public class ParseFixPatternWorker extends UntypedActor {
|
||||
File revFile = msgFile.getRevFile();
|
||||
File prevFile = msgFile.getPrevFile();
|
||||
File diffentryFile = msgFile.getDiffEntryFile();
|
||||
File positionFile = msgFile.getPositionFile();
|
||||
if (revFile.getName().toLowerCase().contains("test")) {
|
||||
// File positionFile = msgFile.getPositionFile();
|
||||
/*if (revFile.getName().toLowerCase().contains("test")) {
|
||||
continue;
|
||||
}
|
||||
}*/
|
||||
FixedViolationHunkParser parser = new FixedViolationHunkParser();
|
||||
|
||||
final ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
@@ -109,13 +109,13 @@ public class ParseFixPatternWorker extends UntypedActor {
|
||||
|
||||
String editScript = parser.getAstEditScripts();
|
||||
if ("".equals(editScript)) {
|
||||
if (parser.resultType == 1) {
|
||||
nullGumTreeResults += countAlarms(positionFile, "");
|
||||
} else if (parser.resultType == 2) {
|
||||
noSourceCodeChanges += countAlarms(positionFile, "");
|
||||
} else if (parser.resultType == 3) {
|
||||
noStatementChanges += countAlarms(positionFile, "");
|
||||
}
|
||||
// if (parser.resultType == 1) {
|
||||
// nullGumTreeResults += countAlarms(positionFile, "");
|
||||
// } else if (parser.resultType == 2) {
|
||||
// noSourceCodeChanges += countAlarms(positionFile, "");
|
||||
// } else if (parser.resultType == 3) {
|
||||
// noStatementChanges += countAlarms(positionFile, "");
|
||||
// }
|
||||
} else {
|
||||
editScripts.append(editScript);
|
||||
patchesSourceCode.append(parser.getPatchesSourceCode());
|
||||
@@ -139,14 +139,14 @@ public class ParseFixPatternWorker extends UntypedActor {
|
||||
}
|
||||
} catch (TimeoutException e) {
|
||||
future.cancel(true);
|
||||
timeouts += countAlarms(positionFile, "#Timeout:");
|
||||
// timeouts += countAlarms(positionFile, "#Timeout:");
|
||||
System.err.println("#Timeout: " + revFile.getName());
|
||||
} catch (InterruptedException e) {
|
||||
timeouts += countAlarms(positionFile, "#TimeInterrupted:");
|
||||
// timeouts += countAlarms(positionFile, "#TimeInterrupted:");
|
||||
System.err.println("#TimeInterrupted: " + revFile.getName());
|
||||
e.printStackTrace();
|
||||
} catch (ExecutionException e) {
|
||||
timeouts += countAlarms(positionFile, "#TimeAborted:");
|
||||
// timeouts += countAlarms(positionFile, "#TimeAborted:");
|
||||
System.err.println("#TimeAborted: " + revFile.getName());
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
|
||||
@@ -2,9 +2,10 @@ package edu.lu.uni.serval.config;
|
||||
|
||||
public class Configuration {
|
||||
|
||||
public static final long SECONDS_TO_WAIT = 300L;
|
||||
public static final long SECONDS_TO_WAIT = 100L;
|
||||
|
||||
public static String ROOT_PATH = "../FPM_Violations/"; // The root path of all output data.
|
||||
// public static String ROOT_PATH = "/Volumes/data/bugStudy/dataset/"; // The root path of all output data.
|
||||
public static String ROOT_PATH = "/Users/anilkoyuncu/bugStudy/dataset/"; // The root path of all output data.
|
||||
|
||||
public static int HUNK_SIZE = 10; // The limitation of source code lines of each DiffEntry, which will be selected as training data.
|
||||
public static final String BUGGY_TREE_SIGNAL = "BUGGY_TREE###"; // The starting signal of the tree of buggy source code .
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package edu.lu.uni.serval.gumtree.regroup;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@@ -13,6 +14,7 @@ import com.github.gumtreediff.tree.ITree;
|
||||
|
||||
import edu.lu.uni.serval.FixPattern.utils.ASTNodeMap;
|
||||
import edu.lu.uni.serval.FixPattern.utils.Checker;
|
||||
import edu.lu.uni.serval.gumtree.GumTreeComparer;
|
||||
import edu.lu.uni.serval.utils.ListSorter;
|
||||
|
||||
/**
|
||||
@@ -22,7 +24,7 @@ import edu.lu.uni.serval.utils.ListSorter;
|
||||
*
|
||||
*/
|
||||
public class HierarchicalRegrouper {
|
||||
|
||||
|
||||
public List<HierarchicalActionSet> regroupGumTreeResults(List<Action> actions) {
|
||||
/*
|
||||
* First, sort actions by their positions.
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
package edu.lu.uni.serval.gumtree.regroup;
|
||||
|
||||
import com.github.gumtreediff.actions.model.*;
|
||||
import com.github.gumtreediff.tree.ITree;
|
||||
import edu.lu.uni.serval.FixPattern.utils.CNodeMap;
|
||||
import edu.lu.uni.serval.gumtree.GumTreeComparer;
|
||||
import edu.lu.uni.serval.utils.ListSorter;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Regroup GumTree results to a hierarchical construction.
|
||||
*
|
||||
* @author kui.liu
|
||||
*
|
||||
*/
|
||||
public class HierarchicalRegrouperForC {
|
||||
|
||||
public static void main(String[] args) {
|
||||
GumTreeComparer com = new GumTreeComparer();
|
||||
File cFile1 = new File("/Users/anilkoyuncu/bugStudy/dataset/GumTreeInput/linux-stable/prevFiles/prev_0a3d00_b404bc_drivers#pci#iov.c");
|
||||
File cFile2 = new File("/Users/anilkoyuncu/bugStudy/dataset/GumTreeInput/linux-stable/revFiles/0a3d00_b404bc_drivers#pci#iov.c");
|
||||
List<Action> action = com.compareTwoFilesWithGumTreeForCCode(cFile1, cFile2);
|
||||
List<HierarchicalActionSet> actionSet = new HierarchicalRegrouperForC().regroupGumTreeResults(action);
|
||||
System.out.println(actionSet);
|
||||
}
|
||||
|
||||
public List<HierarchicalActionSet> regroupGumTreeResults(List<Action> actions) {
|
||||
/*
|
||||
* First, sort actions by their positions.
|
||||
*/
|
||||
// List<Action> actions = new ListSorter<Action>(actionsArgu).sortAscending();
|
||||
// if (actions == null) {
|
||||
// actions = actionsArgu;
|
||||
// }
|
||||
|
||||
/*
|
||||
* Second, group actions by their positions.
|
||||
*/
|
||||
List<HierarchicalActionSet> actionSets = new ArrayList<>();
|
||||
HierarchicalActionSet actionSet = null;
|
||||
for(Action act : actions){
|
||||
Action parentAct = findParentAction(act, actions);
|
||||
if (parentAct == null) {
|
||||
actionSet = createActionSet(act, parentAct, null);
|
||||
actionSets.add(actionSet);
|
||||
} else {
|
||||
if (!addToAactionSet(act, parentAct, actionSets)) {
|
||||
// The index of the parent action in the actions' list is larger than the index of this action.
|
||||
actionSet = createActionSet(act, parentAct, null);
|
||||
actionSets.add(actionSet);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Third, add the subActionSet to its parent ActionSet.
|
||||
*/
|
||||
List<HierarchicalActionSet> reActionSets = new ArrayList<>();
|
||||
for (HierarchicalActionSet actSet : actionSets) {
|
||||
Action parentAct = actSet.getParentAction();
|
||||
if (parentAct != null) {
|
||||
addToActionSets(actSet, parentAct, actionSets);
|
||||
} else {
|
||||
// TypeDeclaration, FieldDeclaration, MethodDeclaration, Statement.
|
||||
// CatchClause, ConstructorInvocation, SuperConstructorInvocation, SwitchCase
|
||||
String astNodeType = actSet.getAstNodeType();
|
||||
// if (astNodeType.endsWith("TypeDeclaration") || astNodeType.endsWith("FieldDeclaration") || astNodeType.endsWith("EnumDeclaration") ||
|
||||
// astNodeType.endsWith("MethodDeclaration") || astNodeType.endsWith("Statement") ||
|
||||
// astNodeType.endsWith("ConstructorInvocation") || astNodeType.endsWith("CatchClause") || astNodeType.endsWith("SwitchCase")) {
|
||||
reActionSets.add(actSet);
|
||||
// }
|
||||
}
|
||||
}
|
||||
return reActionSets;
|
||||
}
|
||||
|
||||
private HierarchicalActionSet createActionSet(Action act, Action parentAct, HierarchicalActionSet parent) {
|
||||
HierarchicalActionSet actionSet = new HierarchicalActionSet();
|
||||
actionSet.setAction(act);
|
||||
actionSet.setActionString(parseAction(act.toString()));
|
||||
actionSet.setParentAction(parentAct);
|
||||
actionSet.setNode(act.getNode());
|
||||
actionSet.setParent(parent);
|
||||
return actionSet;
|
||||
}
|
||||
|
||||
private String parseAction(String actStr1) {
|
||||
// UPD 25@@!a from !a to isTrue(a) at 69
|
||||
String[] actStrArrays = actStr1.split("@@");
|
||||
String actStr = "";
|
||||
int length = actStrArrays.length;
|
||||
for (int i =0; i < length - 1; i ++) {
|
||||
String actStrFrag = actStrArrays[i];
|
||||
int index = actStrFrag.lastIndexOf(" ") + 1;
|
||||
String nodeType = actStrFrag.substring(index);
|
||||
if (!"".equals(nodeType)) {
|
||||
if (Character.isDigit(nodeType.charAt(0)) || (nodeType.startsWith("-") && Character.isDigit(nodeType.charAt(1)))) {
|
||||
try {
|
||||
int typeInt = Integer.parseInt(nodeType);
|
||||
if (CNodeMap.map.containsKey(typeInt)) {
|
||||
String type = CNodeMap.map.get(Integer.parseInt(nodeType));
|
||||
nodeType = type;
|
||||
}
|
||||
} catch (NumberFormatException e) {
|
||||
nodeType = actStrFrag.substring(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
actStrFrag = actStrFrag.substring(0, index) + nodeType + "@@";
|
||||
actStr += actStrFrag;
|
||||
}
|
||||
actStr += actStrArrays[length - 1];
|
||||
return actStr;
|
||||
}
|
||||
|
||||
private void addToActionSets(HierarchicalActionSet actionSet, Action parentAct, List<HierarchicalActionSet> actionSets) {
|
||||
Action act = actionSet.getAction();
|
||||
for (HierarchicalActionSet actSet : actionSets) {
|
||||
if (actSet.equals(actionSet)) continue;
|
||||
Action action = actSet.getAction();
|
||||
|
||||
if (!areRelatedActions(action, act)) continue;
|
||||
if (action.equals(parentAct)) { // actSet is the parent of actionSet.
|
||||
actionSet.setParent(actSet);
|
||||
actSet.getSubActions().add(actionSet);
|
||||
sortSubActions(actSet);
|
||||
break;
|
||||
} else {
|
||||
if (isPossibileSubAction(action, act)) {
|
||||
// SubAction range： startPosition2 <= startPosition && startPosition + length <= startPosition2 + length2
|
||||
addToActionSets(actionSet, parentAct, actSet.getSubActions());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isPossibileSubAction(Action parent, Action child) {
|
||||
if ((parent instanceof Update && !(child instanceof Addition))
|
||||
|| (parent instanceof Delete && child instanceof Delete)
|
||||
|| (parent instanceof Insert && (child instanceof Insert))) {
|
||||
int startPosition = child.getPosition();
|
||||
int length = child.getLength();
|
||||
int startPosition2 = parent.getPosition();
|
||||
int length2 = parent.getLength();
|
||||
|
||||
if (!(startPosition2 <= startPosition && startPosition + length <= startPosition2 + length2)) {
|
||||
// when act is not the sub-set of action.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private void sortSubActions(HierarchicalActionSet actionSet) {
|
||||
ListSorter<HierarchicalActionSet> sorter = new ListSorter<HierarchicalActionSet>(actionSet.getSubActions());
|
||||
List<HierarchicalActionSet> subActions = sorter.sortAscending();
|
||||
if (subActions != null) {
|
||||
actionSet.setSubActions(subActions);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean addToAactionSet(Action act, Action parentAct, List<HierarchicalActionSet> actionSets) {
|
||||
for(HierarchicalActionSet actionSet : actionSets) {
|
||||
Action action = actionSet.getAction();
|
||||
|
||||
if (!areRelatedActions(action, act)) continue;
|
||||
|
||||
if (action.equals(parentAct)) { // actionSet is the parent of actSet.
|
||||
HierarchicalActionSet actSet = createActionSet(act, actionSet.getAction(), actionSet);
|
||||
actionSet.getSubActions().add(actSet);
|
||||
sortSubActions(actionSet);
|
||||
return true;
|
||||
} else {
|
||||
if (isPossibileSubAction(action, act)) {
|
||||
// SubAction range： startPosition2 <= startPosition && startPosition + length <= startP + length2
|
||||
List<HierarchicalActionSet> subActionSets = actionSet.getSubActions();
|
||||
if (subActionSets.size() > 0) {
|
||||
boolean added = addToAactionSet(act, parentAct, subActionSets);
|
||||
if (added) {
|
||||
return true;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private Action findParentAction(Action action, List<Action> actions) {
|
||||
|
||||
ITree parent = action.getNode().getParent();
|
||||
if (action instanceof Addition) {
|
||||
parent = ((Addition) action).getParent(); // parent in the fixed source code tree
|
||||
}
|
||||
|
||||
if (parent.getType() == 55) {
|
||||
int type = action.getNode().getType();
|
||||
// Modifier, NormalAnnotation, MarkerAnnotation, SingleMemberAnnotation
|
||||
if (type != 83 && type != 77 && type != 78 && type != 79
|
||||
&& type != 5 && type != 39 && type != 43 && type != 74 && type != 75
|
||||
&& type != 76 && type != 84 && type != 87 && type != 88 && type != 42) {
|
||||
// ArrayType, PrimitiveType, SimpleType, ParameterizedType,
|
||||
// QualifiedType, WildcardType, UnionType, IntersectionType, NameQualifiedType, SimpleName
|
||||
return null;
|
||||
}
|
||||
// } else if (parent.getType() == 31) { // method declaration
|
||||
// int type = action.getNode().getType();
|
||||
// if (Checker.isStatement(type)) {// statements
|
||||
// return null;
|
||||
// }
|
||||
}
|
||||
|
||||
for (Action act : actions) {
|
||||
if (act.getNode().equals(parent)) {
|
||||
if (areRelatedActions(act, action)) {
|
||||
return act;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean areRelatedActions(Action parent, Action child) {
|
||||
if (parent instanceof Move && !(child instanceof Move)) {// If action is MOV, its children must be MOV.
|
||||
return false;
|
||||
}
|
||||
if (parent instanceof Delete && !(child instanceof Delete)) {// If action is INS, its children must be MOV or INS.
|
||||
return false;
|
||||
}
|
||||
if (parent instanceof Insert && !(child instanceof Addition)) {// If action is DEL, its children must be DEL.
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user