initial commit

This commit is contained in:
Anil Koyuncu
2018-02-21 16:31:58 +01:00
parent c6fa20645b
commit a1e9627566
8 changed files with 167 additions and 131 deletions
-1
View File
@@ -42,7 +42,6 @@
<orderEntry type="library" name="Maven: org.eclipse.equinox:app:1.3.200-v20130910-1609" level="project" /> <orderEntry type="library" name="Maven: org.eclipse.equinox:app:1.3.200-v20130910-1609" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.birt.runtime:org.eclipse.core.resources:3.10.0.v20150423-0755" level="project" /> <orderEntry type="library" name="Maven: org.eclipse.birt.runtime:org.eclipse.core.resources:3.10.0.v20150423-0755" level="project" />
<orderEntry type="library" name="Maven: org.eclipse.tycho:org.eclipse.jdt.core:3.12.2.v20161117-1814" level="project" /> <orderEntry type="library" name="Maven: org.eclipse.tycho:org.eclipse.jdt.core:3.12.2.v20161117-1814" level="project" />
<orderEntry type="library" name="Maven: edu.lu.uni.serval:SourceCodeParser:0.0.1-SNAPSHOT" level="project" />
<orderEntry type="module" module-name="GitTraveller" /> <orderEntry type="module" module-name="GitTraveller" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.7" level="project" /> <orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.7" level="project" />
<orderEntry type="library" name="Maven: org.jsoup:jsoup:1.11.2" level="project" /> <orderEntry type="library" name="Maven: org.jsoup:jsoup:1.11.2" level="project" />
@@ -36,7 +36,8 @@ public abstract class Parser implements ParserInterface {
protected List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree(File prevFile, File revFile) { protected List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree(File prevFile, File revFile) {
List<HierarchicalActionSet> actionSets = new ArrayList<>(); List<HierarchicalActionSet> actionSets = new ArrayList<>();
// GumTree results // GumTree results
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTreeForCCode(prevFile, revFile); List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
// List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTreeForCCode(prevFile, revFile);
if (gumTreeResults != null && gumTreeResults.size() > 0) { if (gumTreeResults != null && gumTreeResults.size() > 0) {
// Regroup GumTre results. // Regroup GumTre results.
List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults); List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
@@ -49,11 +49,11 @@ public class FixedViolationHunkParser extends FixedViolationParser {
// type = "#NoStatementChange:"; // type = "#NoStatementChange:";
// } // }
} else { } else {
List<DiffEntryHunk> diffentryHunks = new DiffEntryReader().readHunks2(diffentryFile); List<DiffEntryHunk> diffentryHunks = new DiffEntryReader().readHunks3(diffentryFile);
//Filter out the modify actions, which are not in the DiffEntry hunks. //Filter out the modify actions, which are not in the DiffEntry hunks.
HunkActionFilter hunkFilter = new HunkActionFilter(); HunkActionFilter hunkFilter = new HunkActionFilter();
List<DiffEntryHunk> selectedPatchHunks = hunkFilter.matchActionsByDiffEntryForC(diffentryHunks, actionSets); List<DiffEntryHunk> selectedPatchHunks = hunkFilter.filterActionsByModifiedRange2(diffentryHunks, actionSets, revFile, prevFile);
for (DiffEntryHunk patchHunk : selectedPatchHunks) { for (DiffEntryHunk patchHunk : selectedPatchHunks) {
List<HierarchicalActionSet> hunkActionSets = patchHunk.getActionSets(); List<HierarchicalActionSet> hunkActionSets = patchHunk.getActionSets();
@@ -69,7 +69,6 @@ public class FixedViolationHunkParser extends FixedViolationParser {
int bugEndPosition = 0; int bugEndPosition = 0;
int fixEndPosition = 0; int fixEndPosition = 0;
for (HierarchicalActionSet hunkActionSet : hunkActionSets) { for (HierarchicalActionSet hunkActionSet : hunkActionSets) {
//TODO FIX ME
int actionBugStart = hunkActionSet.getBugStartLineNum(); int actionBugStart = hunkActionSet.getBugStartLineNum();
int actionBugEnd = hunkActionSet.getBugEndLineNum(); int actionBugEnd = hunkActionSet.getBugEndLineNum();
int actionFixStart = hunkActionSet.getFixStartLineNum(); int actionFixStart = hunkActionSet.getFixStartLineNum();
@@ -100,6 +99,11 @@ public class FixedViolationHunkParser extends FixedViolationParser {
continue; continue;
} }
if (fixStartLine == 0 && bugStartLine != 0) {// pure delete actions.
// get the exact buggy code by violation's position. TODO later
}
// if (children.size() == 0) continue;
boolean isPureInsert = false; boolean isPureInsert = false;
if (bugStartLine == 0 && patchHunk.getBugLineStartNum() > 0) { if (bugStartLine == 0 && patchHunk.getBugLineStartNum() > 0) {
bugStartLine = patchHunk.getBugLineStartNum(); bugStartLine = patchHunk.getBugLineStartNum();
@@ -107,11 +111,13 @@ public class FixedViolationHunkParser extends FixedViolationParser {
isPureInsert = true; isPureInsert = true;
// continue; // continue;
} }
if ((bugEndLine - bugStartLine > Configuration.HUNK_SIZE && !isPureInsert) || fixEndLine - fixStartLine > Configuration.HUNK_SIZE) { // if ((bugEndLine - bugStartLine > Configuration.HUNK_SIZE ) || fixEndLine - fixStartLine > Configuration.HUNK_SIZE) {
// continue; //TODO filter out the // continue; //TODO hunk size
// }
if(patchHunk.getBuggyHunkSize() > Configuration.HUNK_SIZE || patchHunk.getFixedHunkSize() > Configuration.HUNK_SIZE){
continue;
} }
/** /**
* Select edit scripts for deep learning. * Select edit scripts for deep learning.
* Edit scripts will be used to mine common fix patterns. * Edit scripts will be used to mine common fix patterns.
@@ -134,14 +140,18 @@ public class FixedViolationHunkParser extends FixedViolationParser {
String patchPosition = "\n" + revFile.getName() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n"; String patchPosition = "\n" + revFile.getName() + "\n@@ -" + bugStartLine + ", " + bugEndLine + " +" + fixStartLine + ", " + fixEndLine + "@@\n";
String info = Configuration.PATCH_SIGNAL + "\n" + patchPosition + patchHunk.getHunk() + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets, bugEndPosition, fixEndPosition) + "\n"; String info = Configuration.PATCH_SIGNAL + "\n" + patchPosition + patchHunk.getHunk() + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets, bugEndPosition, fixEndPosition) + "\n";
//TODO uncomment the line below for more detailed gumtree input.
// String info = Configuration.PATCH_SIGNAL + "\n" + patchPosition + patchHunk.getHunk() + "\nAST Diff###:\n" + getAstEditScripts(hunkActionSets) + "\n";
// if (noUpdate(editScriptTokens)) {
// }
this.patchesSourceCode += info; this.patchesSourceCode += info;
this.sizes += size + "\n"; this.sizes += size + "\n";
this.astEditScripts += astEditScripts + "\n"; this.astEditScripts += astEditScripts + "\n";
// SimpleTree simpleTree = getBuggyCodeTree(patchHunk, bugEndPosition, prevFile, bugStartLine, bugEndLine); SimpleTree simpleTree = getBuggyCodeTree(patchHunk, bugEndPosition, prevFile, bugStartLine, bugEndLine);
// String tokens = Tokenizer.getTokensDeepFirst(simpleTree).trim(); String tokens = Tokenizer.getTokensDeepFirst(simpleTree).trim();
// this.tokensOfSourceCode += tokens + "\n"; this.tokensOfSourceCode += tokens + "\n";
} }
} }
} }
@@ -12,8 +12,7 @@ import com.github.gumtreediff.actions.model.Action;
import edu.lu.uni.serval.FixPatternParser.Parser; import edu.lu.uni.serval.FixPatternParser.Parser;
import edu.lu.uni.serval.gumtree.GumTreeComparer; import edu.lu.uni.serval.gumtree.GumTreeComparer;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet; import edu.lu.uni.serval.gumtree.regroup.HierarchicalActionSet;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouper;
import edu.lu.uni.serval.gumtree.regroup.HierarchicalRegrouperForC;
import edu.lu.uni.serval.utils.FileHelper; import edu.lu.uni.serval.utils.FileHelper;
import edu.lu.uni.serval.utils.ListSorter; import edu.lu.uni.serval.utils.ListSorter;
@@ -51,7 +50,7 @@ public class FixedViolationParser extends Parser {
protected List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree2(File prevFile, File revFile) { protected List<HierarchicalActionSet> parseChangedSourceCodeWithGumTree2(File prevFile, File revFile) {
List<HierarchicalActionSet> actionSets = new ArrayList<>(); List<HierarchicalActionSet> actionSets = new ArrayList<>();
// GumTree results // GumTree results
List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTreeForCCode(prevFile, revFile); List<Action> gumTreeResults = new GumTreeComparer().compareTwoFilesWithGumTree(prevFile, revFile);
if (gumTreeResults == null) { if (gumTreeResults == null) {
this.resultType = 1; this.resultType = 1;
return null; return null;
@@ -60,7 +59,7 @@ public class FixedViolationParser extends Parser {
return actionSets; return actionSets;
} else { } else {
// Regroup GumTre results. // Regroup GumTre results.
List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouperForC().regroupGumTreeResults(gumTreeResults); List<HierarchicalActionSet> allActionSets = new HierarchicalRegrouper().regroupGumTreeResults(gumTreeResults);
// for (HierarchicalActionSet actionSet : allActionSets) { // for (HierarchicalActionSet actionSet : allActionSets) {
// String astNodeType = actionSet.getAstNodeType(); // String astNodeType = actionSet.getAstNodeType();
// if (astNodeType.endsWith("Statement") || "FieldDeclaration".equals(astNodeType)) { // if (astNodeType.endsWith("Statement") || "FieldDeclaration".equals(astNodeType)) {
@@ -6,111 +6,132 @@ import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import edu.lu.uni.serval.FixPatternParser.RunnableParser; import edu.lu.uni.serval.FixPatternParser.RunnableParser;
import edu.lu.uni.serval.MultipleThreadsParser.MessageFile; import edu.lu.uni.serval.MultipleThreadsParser.MessageFile;
import edu.lu.uni.serval.config.Configuration; import edu.lu.uni.serval.config.Configuration;
import edu.lu.uni.serval.utils.FileHelper; import edu.lu.uni.serval.utils.FileHelper;
import org.eclipse.jgit.revwalk.RevCommit;
public class TestHunkParser { public class TestHunkParser {
public static void main(String[] args) { public static void main(String[] args) {
// input data // input data
String pjName = "linux"; File folder = new File("/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput");
final List<MessageFile> msgFiles = getMessageFiles(Configuration.GUM_TREE_INPUT +pjName +"/"); File[] listOfFiles = folder.listFiles();
System.out.println(msgFiles.size()); Stream<File> stream = Arrays.stream(listOfFiles);
List<File> folders = stream
.filter(x -> !x.getName().startsWith("."))
.collect(Collectors.toList());
// output path List<File> targetList = new ArrayList<File>();
final String editScriptsFilePath = Configuration.EDITSCRIPTS_FILE; for (File f:folders){
final String patchesSourceCodeFilePath = Configuration.PATCH_SOURCECODE_FILE; for(File f1 :f.listFiles()){
final String buggyTokensFilePath = Configuration.BUGGY_CODE_TOKENS_FILE; if (!f1.getName().startsWith(".")){
final String editScriptSizesFilePath = Configuration.EDITSCRIPT_SIZES_FILE; targetList.add(f1);
final String alarmTypesFilePath = Configuration.ALARM_TYPES_FILE; }
}
}
// FileHelper.deleteDirectory(editScriptsFilePath); for (File target : targetList) {
// FileHelper.deleteDirectory(patchesSourceCodeFilePath); final List<MessageFile> msgFiles = getMessageFiles(target.toString() + "/"); //"/Users/anilkoyuncu/bugStudy/code/python/GumTreeInput/Apache/CAMEL/"
// FileHelper.deleteDirectory(buggyTokensFilePath); System.out.println(msgFiles.size());
// FileHelper.deleteDirectory(editScriptSizesFilePath); if(msgFiles.size() == 0)
// FileHelper.deleteDirectory(alarmTypesFilePath); continue;
String pjName = target.getName();
// output path
String GUM_TREE_OUTPUT = Configuration.ROOT_PATH + pjName + "/";
final String editScriptsFilePath = GUM_TREE_OUTPUT + "editScripts.list";
final String patchesSourceCodeFilePath =GUM_TREE_OUTPUT + "patchSourceCode.list";
final String buggyTokensFilePath = GUM_TREE_OUTPUT + "tokens.list";
final String editScriptSizesFilePath = GUM_TREE_OUTPUT + "editScriptSizes.csv";
final String alarmTypesFilePath = GUM_TREE_OUTPUT + "alarmTypes.list";
StringBuilder astEditScripts = new StringBuilder();
StringBuilder tokens = new StringBuilder();
StringBuilder sizes = new StringBuilder();
StringBuilder patches = new StringBuilder();
StringBuilder alarmTypes = new StringBuilder();
int a = 0;
FileHelper.deleteDirectory(editScriptsFilePath);
FileHelper.deleteDirectory(patchesSourceCodeFilePath);
FileHelper.deleteDirectory(buggyTokensFilePath);
FileHelper.deleteDirectory(editScriptSizesFilePath);
FileHelper.deleteDirectory(alarmTypesFilePath);
StringBuilder astEditScripts = new StringBuilder();
StringBuilder tokens = new StringBuilder();
StringBuilder sizes = new StringBuilder();
StringBuilder patches = new StringBuilder();
StringBuilder alarmTypes = new StringBuilder();
int a = 0;
// int counter = 0; // int counter = 0;
for (MessageFile msgFile : msgFiles) { for (MessageFile msgFile : msgFiles) {
FixedViolationHunkParser parser = new FixedViolationHunkParser(); FixedViolationHunkParser parser = new FixedViolationHunkParser();
final ExecutorService executor = Executors.newSingleThreadExecutor(); final ExecutorService executor = Executors.newSingleThreadExecutor();
// schedule the work // schedule the work
final Future<?> future = executor.submit(new RunnableParser(msgFile.getPrevFile(), final Future<?> future = executor.submit(new RunnableParser(msgFile.getPrevFile(),
msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser)); msgFile.getRevFile(), msgFile.getDiffEntryFile(), parser));
try { try {
// where we wait for task to complete // where we wait for task to complete
future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS); future.get(Configuration.SECONDS_TO_WAIT, TimeUnit.SECONDS);
String editScripts = parser.getAstEditScripts(); String editScripts = parser.getAstEditScripts();
if (!editScripts.equals("")) { if (!editScripts.equals("")) {
astEditScripts.append(editScripts); astEditScripts.append(editScripts);
tokens.append(parser.getTokensOfSourceCode()); tokens.append(parser.getTokensOfSourceCode());
sizes.append(parser.getSizes()); sizes.append(parser.getSizes());
patches.append(parser.getPatchesSourceCode()); patches.append(parser.getPatchesSourceCode());
alarmTypes.append(parser.getAlarmTypes()); alarmTypes.append(parser.getAlarmTypes());
a ++; a++;
if (a % 100 == 0) { if (a % 10 == 0) {
FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true);
FileHelper.outputToFile(buggyTokensFilePath, tokens, true); FileHelper.outputToFile(buggyTokensFilePath, tokens, true);
FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); FileHelper.outputToFile(editScriptSizesFilePath, sizes, true);
FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true);
FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true);
astEditScripts.setLength(0); astEditScripts.setLength(0);
tokens.setLength(0); tokens.setLength(0);
sizes.setLength(0); sizes.setLength(0);
patches.setLength(0); patches.setLength(0);
alarmTypes.setLength(0); alarmTypes.setLength(0);
System.out.println("Finish of parsing " + a + " files......"); System.out.println("Finish of parsing " + a + " files......");
} }
} }
} catch (TimeoutException e) { } catch (TimeoutException e) {
err.println("task timed out"); err.println("task timed out");
future.cancel(true /* mayInterruptIfRunning */ ); future.cancel(true /* mayInterruptIfRunning */);
} catch (InterruptedException e) { } catch (InterruptedException e) {
err.println("task interrupted"); err.println("task interrupted");
} catch (ExecutionException e) { } catch (ExecutionException e) {
err.println("task aborted"); err.println("task aborted");
} finally { } finally {
executor.shutdownNow(); executor.shutdownNow();
} }
} }
FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true); FileHelper.outputToFile(editScriptsFilePath, astEditScripts, true);
FileHelper.outputToFile(buggyTokensFilePath, tokens, true); FileHelper.outputToFile(buggyTokensFilePath, tokens, true);
FileHelper.outputToFile(editScriptSizesFilePath, sizes, true); FileHelper.outputToFile(editScriptSizesFilePath, sizes, true);
FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true); FileHelper.outputToFile(patchesSourceCodeFilePath, patches, true);
FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true); FileHelper.outputToFile(alarmTypesFilePath, alarmTypes, true);
astEditScripts.setLength(0); astEditScripts.setLength(0);
tokens.setLength(0); tokens.setLength(0);
sizes.setLength(0); sizes.setLength(0);
patches.setLength(0); patches.setLength(0);
alarmTypes.setLength(0); alarmTypes.setLength(0);
System.out.println(a); System.out.println(a);
// classifyByAlarmTypes(); // classifyByAlarmTypes();
}
} }
@@ -119,21 +140,25 @@ public class TestHunkParser {
File revFilesPath = new File(inputPath + "revFiles/"); File revFilesPath = new File(inputPath + "revFiles/");
File[] revFiles = revFilesPath.listFiles(); // project folders File[] revFiles = revFilesPath.listFiles(); // project folders
List<MessageFile> msgFiles = new ArrayList<>(); List<MessageFile> msgFiles = new ArrayList<>();
if (revFiles.length >= 0) {
for (File revFile : revFiles) { for (File revFile : revFiles) {
// if (revFile.getName().endsWith(".java")) { // if (revFile.getName().endsWith(".java")) {
String fileName = revFile.getName(); String fileName = revFile.getName();
File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file File prevFile = new File(gumTreeInput + "prevFiles/prev_" + fileName);// previous file
fileName = fileName.replace(".java", ".txt"); fileName = fileName.replace(".java", ".txt");
File diffentryFile = new File(gumTreeInput + "diffentries/" + fileName); // DiffEntry file File diffentryFile = new File(gumTreeInput + "DiffEntries/" + fileName); // DiffEntry file
File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file File positionFile = new File(gumTreeInput + "positions/" + fileName); // position file
MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile); MessageFile msgFile = new MessageFile(revFile, prevFile, diffentryFile);
msgFile.setPositionFile(positionFile); msgFile.setPositionFile(positionFile);
msgFiles.add(msgFile); msgFiles.add(msgFile);
// } // }
} }
return msgFiles; return msgFiles;
}
else{
return null;
}
} }
public static void classifyByAlarmTypes() { public static void classifyByAlarmTypes() {
@@ -52,8 +52,8 @@ public class AkkaParser2 {
// String GIT_REPOSITORY_PATH = "/Users/anilkoyuncu/bugLocalizationStudy/irblsensitivity/data/" + pjPath + "/" + pjName + "/gitrepo/.git"; // String GIT_REPOSITORY_PATH = "/Users/anilkoyuncu/bugLocalizationStudy/irblsensitivity/data/" + pjPath + "/" + pjName + "/gitrepo/.git";
log.info(pjName); log.info(pjName);
// input data // input data
log.info("Get the input data..." + Configuration.GUM_TREE_INPUT +pjName +"/"); log.info("Get the input data..." + Configuration.GUM_TREE_INPUT );
final List<MessageFile> msgFiles = getMessageFiles(Configuration.GUM_TREE_INPUT +pjName +"/"); final List<MessageFile> msgFiles = getMessageFiles(Configuration.GUM_TREE_INPUT);
log.info("MessageFiles: " + msgFiles.size()); log.info("MessageFiles: " + msgFiles.size());
// output path // output path
@@ -6,6 +6,7 @@ import java.io.IOException;
import java.io.StringReader; import java.io.StringReader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.regex.Pattern;
import edu.lu.uni.serval.utils.FileHelper; import edu.lu.uni.serval.utils.FileHelper;
@@ -101,6 +102,7 @@ public class DiffEntryReader {
while ((line = reader.readLine()) != null) { while ((line = reader.readLine()) != null) {
if (RegExp.filterSignal(line.trim())) { if (RegExp.filterSignal(line.trim())) {
// line = Pattern.compile("^@@\\s\\-\\d+,*\\d*\\s\\+\\d+,*\\d*\\s@@").split(line)[1];
sourceCode = true; sourceCode = true;
if (hunk.length() > 0) { if (hunk.length() > 0) {
if (startLine > 0) { if (startLine > 0) {