azalea/fixminer_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[O] Reformat, optimize, add comments

Browse Source
This commit is contained in:
Azalea (on HyDEV-Daisy)
2022-05-08 23:54:51 -04:00
parent 5eca74116e
commit bfbd5f8a2a
4 changed files with 275 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/commitCollector.py
+27 -17
View File
@@ -1,4 +1,6 @@
import json
import pandas as pd
from common.commons import * from common.commons import *
@@ -7,46 +9,54 @@ DATA_PATH = os.environ["DATA_PATH"]
COMMIT_DFS = os.environ["COMMIT_DFS"] COMMIT_DFS = os.environ["COMMIT_DFS"]
COMMIT_FOLDER = os.environ["COMMIT_FOLDER"] COMMIT_FOLDER = os.environ["COMMIT_FOLDER"]
def getCommitFromRepo(f,gitrepo,branch):
cmd = 'git -C ' + f + ' checkout -f ' + branch
output, err = shellGitCheckout(cmd) def getCommitFromRepo(f: PathLike, gitrepo: str, branch: str):
"""
:param f: Git repo directory
:param gitrepo: Repo name
:param branch: Branch name
:return: None
"""
file = f'{gitrepo}.commits'
output, err = shellGitCheckout(f'git -C {f} checkout -f {branch}')
m = re.search(branch, err) m = re.search(branch, err)
while not m: while not m:
time.sleep(10) time.sleep(10)
logging.info('Waiting for checkout') logging.info('Waiting for checkout')
cmd = 'git -C ' + f + " log --no-merges --pretty=format:'{\"commit\":\"%H\",\"commitDate\":\"%ci\",\"title\":\"%f\",\"committer\":\"%ce\"}' > " + gitrepo + '.commits'
output = shellCallTemplate(cmd,enc='latin1')
# Create commits file
form = json.dumps({"commit": "%H", "commitDate": "%ci", "title": "%f", "committer": "%ce"})
shellCallTemplate(f"git -C {f} log --no-merges --pretty=format:'{form}' > {file}", enc='latin1')
def makeDF(filename): # Collect commits
with open(filename,encoding='latin1') as f: commits = json.loads(f'[{Path(file).read_text()}]')
lines = f.readlines()
ls = [eval(f) for f in lines] # Convert to DataFrame
ds = pd.DataFrame.from_dict(ls) ds = pd.DataFrame.from_dict(commits)
ds['commitDate']= ds['commitDate'].apply(lambda x:pd.to_datetime(x)) ds['commitDate'] = pd.to_datetime(ds['commitDate'])
return ds return ds
def caseCollect(subject): def caseCollect(subject):
if not os.path.exists(COMMIT_FOLDER): if not os.path.exists(COMMIT_FOLDER):
os.mkdir(COMMIT_FOLDER) os.mkdir(COMMIT_FOLDER)
if not os.path.exists(COMMIT_DFS): if not os.path.exists(COMMIT_DFS):
os.mkdir(COMMIT_DFS) os.mkdir(COMMIT_DFS)
subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv')) subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
if subject == 'ALL': if subject == 'ALL':
tuples = subjects[['Repo', 'Branch']].values.tolist() tuples = subjects[['Repo', 'Branch']].values.tolist()
else: else:
# repos = subjects.query("Subject == '{0}'".format(subject)).Repo.tolist() # repos = subjects.query("Subject == '{0}'".format(subject)).Repo.tolist()
tuples = subjects.query("Subject == '{0}'".format(subject))[['Repo', 'Branch']].values.tolist() tuples = subjects.query("Subject == '{0}'".format(subject))[
['Repo', 'Branch']].values.tolist()
for t in tuples: for t in tuples:
repo,branch = t repo, branch = t
logging.info(repo) logging.info(repo)
getCommitFromRepo(join(REPO_PATH, repo), join(COMMIT_FOLDER, repo),branch) getCommitFromRepo(join(REPO_PATH, repo), join(COMMIT_FOLDER, repo), branch)
if subject == 'ALL': if subject == 'ALL':
commits = listdir(COMMIT_FOLDER) commits = listdir(COMMIT_FOLDER)
@@ -60,6 +70,7 @@ def caseCollect(subject):
save_zipped_pickle(rDF, join(COMMIT_DFS, repoName + ".pickle")) save_zipped_pickle(rDF, join(COMMIT_DFS, repoName + ".pickle"))
# p.dump(rDF, open(join(COMMIT_DFS, repoName + ".pickle"), "wb")) # p.dump(rDF, open(join(COMMIT_DFS, repoName + ".pickle"), "wb"))
def caseClone(subject): def caseClone(subject):
if not os.path.exists(REPO_PATH): if not os.path.exists(REPO_PATH):
os.mkdir(REPO_PATH) os.mkdir(REPO_PATH)
@@ -74,4 +85,3 @@ def caseClone(subject):
for gitrepo in gitrepos: for gitrepo in gitrepos:
cmd = 'git clone ' + gitrepo cmd = 'git clone ' + gitrepo
out = shellCallTemplate(cmd) out = shellCallTemplate(cmd)
python/common/commons.py
+119 -112
View File
@@ -1,7 +1,8 @@
import logging import logging
import sys import sys
import gzip import gzip
from typing import Union
import numpy as np import numpy as np
from tqdm import tqdm from tqdm import tqdm
import shutil import shutil
@@ -28,16 +29,18 @@ import datetime
import subprocess import subprocess
from pathlib import Path from pathlib import Path
PathLike = Union[os.PathLike, str]
sourceCodeColumns = ['packageName', 'className', 'methodNames', 'formalParameter', sourceCodeColumns = ['packageName', 'className', 'methodNames', 'formalParameter',
'methodInvocation', 'memberReference', 'documentation', 'literal', 'rawSource', 'hunks', 'methodInvocation', 'memberReference', 'documentation', 'literal', 'rawSource',
'commitLogs', 'classNameExt'] 'hunks',
'commitLogs', 'classNameExt']
def nap(): def nap():
time.sleep(1) time.sleep(1)
def setLogg(): def setLogg():
# logging.basicConfig(filename='app.log', filemode='w',level=logging.DEBUG) # logging.basicConfig(filename='app.log', filemode='w',level=logging.DEBUG)
root = logging.getLogger() root = logging.getLogger()
@@ -45,7 +48,8 @@ def setLogg():
ch = logging.StreamHandler(sys.stdout) ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.WARNING) ch.setLevel(logging.WARNING)
formatter = logging.Formatter('%(asctime)s - %(process)d - %(levelname)s - %(filename)s:%(funcName)s - %(message)s') formatter = logging.Formatter(
'%(asctime)s - %(process)d - %(levelname)s - %(filename)s:%(funcName)s - %(message)s')
ch.setFormatter(formatter) ch.setFormatter(formatter)
# ch.addFilter(lambda record: record.levelno <= logging.) # ch.addFilter(lambda record: record.levelno <= logging.)
root.addHandler(ch) root.addHandler(ch)
@@ -64,6 +68,7 @@ def setLogg():
h2.setFormatter(formatter) h2.setFormatter(formatter)
root.addHandler(h2) root.addHandler(h2)
def setEnv(args): def setEnv(args):
# env = args.env # env = args.env
@@ -72,7 +77,6 @@ def setEnv(args):
os.environ["ROOT_DIR"] = args.root os.environ["ROOT_DIR"] = args.root
sys.path.append(args.root) sys.path.append(args.root)
import yaml import yaml
# if os.uname().nodename != '': # if os.uname().nodename != '':
# with open(join(os.environ["ROOT_DIR"], os.uname().nodename + ".config.yml"), 'r') as ymlfile: # with open(join(os.environ["ROOT_DIR"], os.uname().nodename + ".config.yml"), 'r') as ymlfile:
@@ -113,16 +117,15 @@ def setEnv(args):
# os.environ["JDK8"] = cfg['java']['8home'] # os.environ["JDK8"] = cfg['java']['8home']
# os.environ["D4JHOME"] = cfg['defects4j']['home'] # os.environ["D4JHOME"] = cfg['defects4j']['home']
os.environ["CODE_PATH"] = join(os.environ["ROOT_DIR"], 'code/')
os.environ["CODE_PATH"] = join(os.environ["ROOT_DIR"],'code/')
# os.environ["DATA_PATH"] = join(os.environ["ROOT_DIR"],'data/') # os.environ["DATA_PATH"] = join(os.environ["ROOT_DIR"],'data/')
# os.environ["REPO_PATH"] = join(os.environ["DATA_PATH"], 'gitrepo/') # os.environ["REPO_PATH"] = join(os.environ["DATA_PATH"], 'gitrepo/')
os.environ["COMMIT_DFS"]= join(os.environ["DATA_PATH"],'commitsDF/') os.environ["COMMIT_DFS"] = join(os.environ["DATA_PATH"], 'commitsDF/')
os.environ["SIMI_DIR"]= join(os.environ["DATA_PATH"],'simi/') os.environ["SIMI_DIR"] = join(os.environ["DATA_PATH"], 'simi/')
os.environ["DTM_PATH"] = join(os.environ["DATA_PATH"], 'dtm/') os.environ["DTM_PATH"] = join(os.environ["DATA_PATH"], 'dtm/')
os.environ["SIMI_SINGLE"] = join(os.environ["DATA_PATH"], 'simiSingle/') os.environ["SIMI_SINGLE"] = join(os.environ["DATA_PATH"], 'simiSingle/')
os.environ["FEATURE_DIR"] = join(os.environ["DATA_PATH"],'features/') os.environ["FEATURE_DIR"] = join(os.environ["DATA_PATH"], 'features/')
os.environ["BUG_POINT"] = join(os.environ["DATA_PATH"], 'bugPoints/') os.environ["BUG_POINT"] = join(os.environ["DATA_PATH"], 'bugPoints/')
os.environ["DEFECTS4J"] = join(os.environ["DATA_PATH"], 'defects4jdata/') os.environ["DEFECTS4J"] = join(os.environ["DATA_PATH"], 'defects4jdata/')
@@ -139,10 +142,6 @@ def setEnv(args):
os.environ["DATASET_DIR"] = join(os.environ["DATA_PATH"], 'datasets/') os.environ["DATASET_DIR"] = join(os.environ["DATA_PATH"], 'datasets/')
os.environ["REMOTE_PATH"] = '/Volumes/Samsung_T5/data' os.environ["REMOTE_PATH"] = '/Volumes/Samsung_T5/data'
logging.info('ROOT_DIR : %s', os.environ["ROOT_DIR"]) logging.info('ROOT_DIR : %s', os.environ["ROOT_DIR"])
logging.info('REPO_PATH : %s', os.environ["REPO_PATH"]) logging.info('REPO_PATH : %s', os.environ["REPO_PATH"])
logging.info('CODE_PATH : %s', os.environ["CODE_PATH"]) logging.info('CODE_PATH : %s', os.environ["CODE_PATH"])
@@ -159,15 +158,13 @@ def setEnv(args):
logging.info('DATASET_DIR : %s', os.environ["DATASET_DIR"]) logging.info('DATASET_DIR : %s', os.environ["DATASET_DIR"])
def getRun(): def getRun():
import argparse import argparse
parser = argparse.ArgumentParser(description='') parser = argparse.ArgumentParser(description='')
# parser.add_argument('-subject', dest='subject', help='Environment') # parser.add_argument('-subject', dest='subject', help='Environment')
parser.add_argument('-root', dest='root', help='root folder') parser.add_argument('-root', dest='root', help='root folder')
parser.add_argument('-job',dest='job',help='job name') parser.add_argument('-job', dest='job', help='job name')
parser.add_argument('-prop',dest='prop',help='property file') parser.add_argument('-prop', dest='prop', help='property file')
args = parser.parse_args() args = parser.parse_args()
@@ -177,10 +174,9 @@ def getRun():
return args return args
def shellCallTemplate4jar(cmd, enc='utf-8'):
def shellCallTemplate4jar(cmd,enc='utf-8'):
process = subprocess.Popen(cmd, process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,stderr=PIPE, shell=True,encoding=enc, stdout=subprocess.PIPE, stderr=PIPE, shell=True, encoding=enc,
universal_newlines=True) universal_newlines=True)
while True: while True:
@@ -195,10 +191,11 @@ def shellCallTemplate4jar(cmd,enc='utf-8'):
print(output.strip()) print(output.strip())
break break
def shellCallTemplate(cmd,enc='utf-8'):
def shellCallTemplate(cmd, enc='utf-8'):
try: try:
logging.info(cmd) logging.info(cmd)
with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True,encoding=enc) as p: with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, encoding=enc) as p:
output, errors = p.communicate() output, errors = p.communicate()
# print(output) # print(output)
if errors: if errors:
@@ -212,30 +209,32 @@ def shellCallTemplate(cmd,enc='utf-8'):
logging.error(e) logging.error(e)
return output return output
def getChildMem(pid,children):
def getChildMem(pid, children):
out = subprocess.Popen(['pgrep', '-P', str(pid)], out = subprocess.Popen(['pgrep', '-P', str(pid)],
stdout=subprocess.PIPE).communicate()[0].split(b'\n') stdout=subprocess.PIPE).communicate()[0].split(b'\n')
child = out[0].decode() child = out[0].decode()
if child !='': if child != '':
children.append(child) children.append(child)
getChildMem(child,children) getChildMem(child, children)
else: else:
return children return children
def getAllChildMe(pid):
def getAllChildMe(pid):
childrenProcess = [] childrenProcess = []
getChildMem(pid,childrenProcess) getChildMem(pid, childrenProcess)
# if child == '': # if child == '':
return sum(map(memory_usage_ps,childrenProcess)) + memory_usage_ps(pid) return sum(map(memory_usage_ps, childrenProcess)) + memory_usage_ps(pid)
# else: # else:
# return memory_usage_ps(child) + memory_usage_ps(pid) # return memory_usage_ps(child) + memory_usage_ps(pid)
def memory_usage_ps(pid): def memory_usage_ps(pid):
import subprocess import subprocess
out = subprocess.Popen(['ps', 'v', '-p', str(pid)], out = subprocess.Popen(['ps', 'v', '-p', str(pid)],
stdout=subprocess.PIPE).communicate()[0].split(b'\n') stdout=subprocess.PIPE).communicate()[0].split(b'\n')
vsz_index = out[0].split().index(b'RSS') vsz_index = out[0].split().index(b'RSS')
if out[1].decode() != '': if out[1].decode() != '':
mem = float(out[1].split()[vsz_index]) / 1024 mem = float(out[1].split()[vsz_index]) / 1024
@@ -243,73 +242,74 @@ def memory_usage_ps(pid):
mem = float(0) mem = float(0)
return mem return mem
def raiseTime(cmd,timeout,my_timer):
def raiseTime(cmd, timeout, my_timer):
my_timer.cancel() my_timer.cancel()
raise TimeoutExpired(cmd, timeout) raise TimeoutExpired(cmd, timeout)
def killP(pid): def killP(pid):
out = subprocess.Popen(['kill', str(pid)], out = subprocess.Popen(['kill', str(pid)], stdout=subprocess.PIPE).communicate()[0].split(b'\n')
stdout=subprocess.PIPE).communicate()[0].split(b'\n')
out out
def shellGitCheckout(cmd,timeout =600,enc='utf-8'): def shellGitCheckout(cmd, timeout=600, enc='utf-8'):
output = '' output = ''
errors = '' errors = ''
# logging.debug(cmd) # logging.debug(cmd)
with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True,encoding=enc) as p: with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, encoding=enc) as p:
try: try:
output, errors = p.communicate(timeout=timeout) output, errors = p.communicate(timeout=timeout)
# print(output) # print(output)
logging.debug(cmd + '\t' +output) logging.debug(cmd + '\t' + output)
# logging.info(errors) # logging.info(errors)
if errors: if errors:
raise CalledProcessError(errors, '-1') raise CalledProcessError(errors, '-1')
output output
except CalledProcessError as e: except CalledProcessError as e:
logging.debug(cmd +'\t'+ errors) logging.debug(cmd + '\t' + errors)
except TimeoutExpired as t: except TimeoutExpired as t:
p.terminate() p.terminate()
p.communicate() p.communicate()
# p.kill() # p.kill()
logging.warning(cmd +'\t'+str(t)) logging.warning(cmd + '\t' + str(t))
return output,errors return output, errors
def callSpinfer(cmd,timeout =600,enc='utf-8'):
def callSpinfer(cmd, timeout=600, enc='utf-8'):
output = '' output = ''
errors = '' errors = ''
# logging.debug(cmd) # logging.debug(cmd)
my_timer = None my_timer = None
with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True,encoding=enc) as p: with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, encoding=enc) as p:
try: try:
start = datetime.datetime.now() start = datetime.datetime.now()
memusage = getAllChildMe(p.pid) memusage = getAllChildMe(p.pid)
# isExit = False # isExit = False
while(memusage != 0.0): while (memusage != 0.0):
end = datetime.datetime.now() end = datetime.datetime.now()
elapsed = end - start elapsed = end - start
if(elapsed.seconds > timeout): if (elapsed.seconds > timeout):
raise TimeoutExpired(cmd,timeout) raise TimeoutExpired(cmd, timeout)
memusage = getAllChildMe(p.pid) memusage = getAllChildMe(p.pid)
# print(str(p.pid) + " ; " + str(memusage)) # print(str(p.pid) + " ; " + str(memusage))
if memusage > 2000: if memusage > 2000:
# isExit = True # isExit = True
raise TimeoutExpired(cmd,timeout) raise TimeoutExpired(cmd, timeout)
output, errors = p.communicate(timeout=timeout) output, errors = p.communicate(timeout=timeout)
# print(output) # print(output)
logging.debug(cmd + '\t' +output) logging.debug(cmd + '\t' + output)
# logging.info(errors) # logging.info(errors)
if errors: if errors:
raise CalledProcessError(errors, '-1') raise CalledProcessError(errors, '-1')
output output
except CalledProcessError as e: except CalledProcessError as e:
logging.debug(cmd +'\t'+ errors) logging.debug(cmd + '\t' + errors)
except TimeoutExpired as t: except TimeoutExpired as t:
# my_timer.cancel() # my_timer.cancel()
childrenProcess = [] childrenProcess = []
getChildMem(p.pid, childrenProcess) getChildMem(p.pid, childrenProcess)
[killP(i) for i in childrenProcess] [killP(i) for i in childrenProcess]
@@ -317,30 +317,35 @@ def callSpinfer(cmd,timeout =600,enc='utf-8'):
p.terminate() p.terminate()
p.communicate() p.communicate()
# p.kill() # p.kill()
logging.warning(cmd +'\t'+str(t)) logging.warning(cmd + '\t' + str(t))
return output,errors return output, errors
def save_zipped_pickle(obj, filename, protocol=-1): def save_zipped_pickle(obj, filename, protocol=-1):
with gzip.open(filename, 'wb') as f: with gzip.open(filename, 'wb') as f:
p.dump(obj, f, protocol) p.dump(obj, f, protocol)
def load_zipped_pickle(filename): def load_zipped_pickle(filename):
with gzip.open(filename, 'rb') as f: with gzip.open(filename, 'rb') as f:
loaded_object = p.load(f) loaded_object = p.load(f)
return loaded_object return loaded_object
def file2path(file): def file2path(file):
count = file.count(".") - 1 count = file.count(".") - 1
file = file.replace('.', '/', count) file = file.replace('.', '/', count)
return file return file
def isFileInList(file,checkList):
def isFileInList(file, checkList):
for f in checkList: for f in checkList:
if f in file: if f in file:
return True return True
return False return False
# [i for i in ansFiles if 'org/fusesource/esb/itests/basic/fabric/EsbFeatureTest.java' in i] # [i for i in ansFiles if 'org/fusesource/esb/itests/basic/fabric/EsbFeatureTest.java' in i]
def get_venn_sections(sets): def get_venn_sections(sets):
""" """
Given a list of sets, return a new list of sets with all the possible Given a list of sets, return a new list of sets with all the possible
@@ -366,7 +371,7 @@ def get_venn_sections(sets):
bit_flags = [2 ** n for n in range(len(sets))] bit_flags = [2 ** n for n in range(len(sets))]
flags_zip_sets = [z for z in zip(bit_flags, sets)] flags_zip_sets = [z for z in zip(bit_flags, sets)]
#combo_sets = [] # combo_sets = []
combo_sets = dict() combo_sets = dict()
for bits in range(num_combinations - 1, 0, -1): for bits in range(num_combinations - 1, 0, -1):
include_sets = [s for flag, s in flags_zip_sets if bits & flag] include_sets = [s for flag, s in flags_zip_sets if bits & flag]
@@ -374,17 +379,19 @@ def get_venn_sections(sets):
combo = set.intersection(*include_sets) combo = set.intersection(*include_sets)
combo = set.difference(combo, *exclude_sets) combo = set.difference(combo, *exclude_sets)
tag = ''.join([str(int((bits & flag) > 0)) for flag in bit_flags]) tag = ''.join([str(int((bits & flag) > 0)) for flag in bit_flags])
#combo_sets.append((tag, combo)) # combo_sets.append((tag, combo))
combo_sets[tag] = combo combo_sets[tag] = combo
return combo_sets return combo_sets
def pairwise(iterable): def pairwise(iterable):
"s -> (s0,s1), (s1,s2), (s2, s3), ..." "s -> (s0,s1), (s1,s2), (s2, s3), ..."
a, b = itertools.tee(iterable) a, b = itertools.tee(iterable)
next(b, None) next(b, None)
return zip(a, b) return zip(a, b)
def RR_XGB(x,ao,column):
def RR_XGB(x, ao, column):
if x[ao] == 1: if x[ao] == 1:
return (1.0 / (x[column])) return (1.0 / (x[column]))
elif pd.isnull(x[ao]): elif pd.isnull(x[ao]):
@@ -392,10 +399,11 @@ def RR_XGB(x,ao,column):
else: else:
return 0 return 0
def parallelRunNo(coreFun,elements,*args):
def parallelRunNo(coreFun, elements, *args):
with concurrent.futures.ProcessPoolExecutor(max_workers=int(8)) as executor: with concurrent.futures.ProcessPoolExecutor(max_workers=int(8)) as executor:
try: try:
futures = {executor.submit(coreFun, l,*args): l for l in elements} futures = {executor.submit(coreFun, l, *args): l for l in elements}
kwargs = { kwargs = {
'total': len(futures), 'total': len(futures),
@@ -420,10 +428,10 @@ def parallelRunNo(coreFun,elements,*args):
raise raise
def parallelRun(coreFun,elements,*args,max_workers=os.cpu_count()): def parallelRun(coreFun, elements, *args, max_workers=os.cpu_count()):
with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
try: try:
futures = {executor.submit(coreFun, l,*args): l for l in elements} futures = {executor.submit(coreFun, l, *args): l for l in elements}
kwargs = { kwargs = {
'total': len(futures), 'total': len(futures),
@@ -445,11 +453,11 @@ def parallelRun(coreFun,elements,*args,max_workers=os.cpu_count()):
raise raise
def parallelRunMerge(coreFun,elements,*args,max_workers=os.cpu_count()): def parallelRunMerge(coreFun, elements, *args, max_workers=os.cpu_count()):
dataL = [] dataL = []
with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
try: try:
futures = {executor.submit(coreFun, l,*args): l for l in elements} futures = {executor.submit(coreFun, l, *args): l for l in elements}
kwargs = { kwargs = {
'total': len(futures), 'total': len(futures),
'unit': 'files', 'unit': 'files',
@@ -473,12 +481,11 @@ def parallelRunMerge(coreFun,elements,*args,max_workers=os.cpu_count()):
raise raise
def parallelRunMergeNew(coreFun,elements,*args,max_workers=os.cpu_count()): def parallelRunMergeNew(coreFun, elements, *args, max_workers=os.cpu_count()):
res = [] res = []
with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
try: try:
futures = {executor.submit(coreFun, l,*args): l for l in elements} futures = {executor.submit(coreFun, l, *args): l for l in elements}
kwargs = { kwargs = {
'total': len(futures), 'total': len(futures),
@@ -503,8 +510,8 @@ def parallelRunMergeNew(coreFun,elements,*args,max_workers=os.cpu_count()):
aDF = pd.concat(res) aDF = pd.concat(res)
return aDF return aDF
def get_filepaths(directory,extension):
def get_filepaths(directory, extension):
file_paths = [] # List which will store all of the full filepaths.\n, file_paths = [] # List which will store all of the full filepaths.\n,
exclude = '.git' exclude = '.git'
# Walk the tree.\n, # Walk the tree.\n,
@@ -520,27 +527,27 @@ def get_filepaths(directory,extension):
return file_paths # Self-explanatory.\n, return file_paths # Self-explanatory.\n,
def get_class_weights(y): def get_class_weights(y):
counter = Counter(y) counter = Counter(y)
majority = max(counter.values()) majority = max(counter.values())
return {cls: round(float(majority)/float(count), 2) for cls, count in counter.items()} return {cls: round(float(majority) / float(count), 2) for cls, count in counter.items()}
def stopDB(dbDir,portInner): def stopDB(dbDir, portInner):
# cmd = "bash " + dbDir + "/" + "stopServer.sh " + " " + portInner; # cmd = "bash " + dbDir + "/" + "stopServer.sh " + " " + portInner;
cmd = "redis-cli -p " + portInner + " shutdown save" cmd = "redis-cli -p " + portInner + " shutdown save"
o, e = shellGitCheckout(cmd) o, e = shellGitCheckout(cmd)
logging.info(o) logging.info(o)
def startDB(dbDir, portInner, projectType):
def startDB(dbDir,portInner,projectType): dbName = "dumps-" + projectType + ".rdb"
dbName = "dumps-"+projectType+".rdb"
# portInner = '6380' # portInner = '6380'
cmd = "bash " + dbDir + "/" + "startServer.sh " + dbDir + " "+dbName+ " " + portInner; cmd = "bash " + dbDir + "/" + "startServer.sh " + dbDir + " " + dbName + " " + portInner;
o, e = shellGitCheckout(cmd) o, e = shellGitCheckout(cmd)
ping = "redis-cli -p "+portInner+" ping" ping = "redis-cli -p " + portInner + " ping"
o, e = shellGitCheckout(ping) o, e = shellGitCheckout(ping)
m = re.search('PONG', o) m = re.search('PONG', o)
@@ -569,23 +576,23 @@ def unique_everseen(iterable, key=None):
seen_add(k) seen_add(k)
yield element yield element
def plotBox(yList,labels, fn, rotate=False,limit=True):
def plotBox(yList, labels, fn, rotate=False, limit=True):
import matplotlib import matplotlib
matplotlib.use("TkAgg") matplotlib.use("TkAgg")
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
fig = plt.figure() fig = plt.figure()
ax1 = fig.add_subplot(111) ax1 = fig.add_subplot(111)
meanpointsprops = dict(markeredgecolor ='blue',markerfacecolor= meanpointsprops = dict(markeredgecolor='blue', markerfacecolor=
'blue') 'blue')
flierprops = dict(markeredgecolor ='black',markerfacecolor= flierprops = dict(markeredgecolor='black', markerfacecolor=
'black',marker='.',markersize=2) 'black', marker='.', markersize=2)
box = ax1.boxplot(yList, 0, flierprops=flierprops,widths=0.5, showmeans=False, vert=True,meanprops=meanpointsprops) box = ax1.boxplot(yList, 0, flierprops=flierprops, widths=0.5, showmeans=False, vert=True,
meanprops=meanpointsprops)
for line in box['medians']: for line in box['medians']:
x,y = line.get_xydata()[1] x, y = line.get_xydata()[1]
line.set(linewidth=3) line.set(linewidth=3)
line.set_color('blue') line.set_color('blue')
# plt.scatter(labels, yList, color='r') # plt.scatter(labels, yList, color='r')
@@ -601,8 +608,8 @@ def plotBox(yList,labels, fn, rotate=False,limit=True):
ax1.get_xaxis().set_ticklabels([]) ax1.get_xaxis().set_ticklabels([])
# sns.boxplot(yList, ax=ax1) # sns.boxplot(yList, ax=ax1)
if limit: if limit:
ax1.set_ylim(top=1.1,bottom=0) ax1.set_ylim(top=1.1, bottom=0)
ax1.yaxis.set_ticks([0.0,1.0]) ax1.yaxis.set_ticks([0.0, 1.0])
else: else:
ax1.set_yscale('log') ax1.set_yscale('log')
ax1.set_xlabel('Cluster Member Size') ax1.set_xlabel('Cluster Member Size')
@@ -616,33 +623,32 @@ def plotBox(yList,labels, fn, rotate=False,limit=True):
fig.set_size_inches(7, 1, forward=True) fig.set_size_inches(7, 1, forward=True)
fig.savefig(fn, dpi=100, bbox_inches='tight') fig.savefig(fn, dpi=100, bbox_inches='tight')
plt.show() plt.show()
def plotBox2(ys,labels, fn,means, rotate=False,limit=True):
def plotBox2(ys, labels, fn, means, rotate=False, limit=True):
import matplotlib import matplotlib
matplotlib.use("TkAgg") matplotlib.use("TkAgg")
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
fig, axes = plt.subplots(nrows=3, ncols=1)
fig,axes = plt.subplots(nrows=3,ncols=1) for ax1, yList, l, l2, mean in zip(axes.flat, ys, labels, ['Shapes', 'Actions', 'Tokens'],
means):
for ax1,yList,l,l2,mean in zip(axes.flat,ys,labels,['Shapes','Actions','Tokens'],means):
# plt.setp(ax1.get_xticks(),visible=False) # plt.setp(ax1.get_xticks(),visible=False)
# ax1 = fig.add_subplot(111) # ax1 = fig.add_subplot(111)
meanpointsprops = dict(markeredgecolor ='blue',markerfacecolor= meanpointsprops = dict(markeredgecolor='blue', markerfacecolor=
'blue') 'blue')
flierprops = dict(markeredgecolor ='black',markerfacecolor= flierprops = dict(markeredgecolor='black', markerfacecolor=
'black',marker='.',markersize=2) 'black', marker='.', markersize=2)
box = ax1.boxplot(yList, 0, flierprops=flierprops,widths=0.5, showmeans=False, vert=True,meanprops=meanpointsprops) box = ax1.boxplot(yList, 0, flierprops=flierprops, widths=0.5, showmeans=False, vert=True,
meanprops=meanpointsprops)
ax1.axhline(linewidth=2, color='r',y=mean) ax1.axhline(linewidth=2, color='r', y=mean)
for line in box['medians']: for line in box['medians']:
x,y = line.get_xydata()[1] x, y = line.get_xydata()[1]
line.set(linewidth=3) line.set(linewidth=3)
line.set_color('blue') line.set_color('blue')
# plt.scatter(labels, yList, color='r') # plt.scatter(labels, yList, color='r')
@@ -659,14 +665,14 @@ def plotBox2(ys,labels, fn,means, rotate=False,limit=True):
# ax1.get_xaxis().set_ticks([]) # ax1.get_xaxis().set_ticks([])
# sns.boxplot(yList, ax=ax1) # sns.boxplot(yList, ax=ax1)
if limit: if limit:
if l2 !='Tokens': if l2 != 'Tokens':
ax1.set_ylim(top=1,bottom=0) ax1.set_ylim(top=1, bottom=0)
else: else:
ax1.set_ylim(top=1.1, bottom=0) ax1.set_ylim(top=1.1, bottom=0)
ax1.yaxis.set_ticks([0.0,mean,0.5,1.0]) ax1.yaxis.set_ticks([0.0, mean, 0.5, 1.0])
ax1.yaxis.set_ticklabels([0,'',0.5,1]) ax1.yaxis.set_ticklabels([0, '', 0.5, 1])
ax1.tick_params(direction='out', length=6, width=2, axis='y', ax1.tick_params(direction='out', length=6, width=2, axis='y',
grid_color='r', grid_alpha=0.5) grid_color='r', grid_alpha=0.5)
else: else:
# ax1.set_yscale('log') # ax1.set_yscale('log')
@@ -675,7 +681,7 @@ def plotBox2(ys,labels, fn,means, rotate=False,limit=True):
ax1.set_aspect('auto') ax1.set_aspect('auto')
ax1.set_ylabel(l2) ax1.set_ylabel(l2)
labels = ['C-'+str(i+1) for i in labels[0]] labels = ['C-' + str(i + 1) for i in labels[0]]
ax1.set_xticklabels(labels) ax1.set_xticklabels(labels)
ax1.set_xticklabels(labels, rotation=45, ha='right') ax1.set_xticklabels(labels, rotation=45, ha='right')
# plt.setp(ax1.get_xticks(), visible=True) # plt.setp(ax1.get_xticks(), visible=True)
@@ -687,16 +693,14 @@ def plotBox2(ys,labels, fn,means, rotate=False,limit=True):
plt.subplots_adjust(wspace=0, hspace=0.05) plt.subplots_adjust(wspace=0, hspace=0.05)
fig = plt.gcf() fig = plt.gcf()
# fig.tight_layout() # fig.tight_layout()
fig.set_size_inches(7, 7, forward=True) fig.set_size_inches(7, 7, forward=True)
fig.savefig(fn, dpi=100, bbox_inches='tight') fig.savefig(fn, dpi=100, bbox_inches='tight')
plt.show() plt.show()
def plotScatter(s1,s2,vs,label,limits,type): def plotScatter(s1, s2, vs, label, limits, type):
import matplotlib import matplotlib
matplotlib.use("TkAgg") matplotlib.use("TkAgg")
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
@@ -719,8 +723,8 @@ def plotScatter(s1,s2,vs,label,limits,type):
stepsize = 1 stepsize = 1
ax.xaxis.set_ticks(np.arange(0, end, stepsize)) ax.xaxis.set_ticks(np.arange(0, end, stepsize))
ax.yaxis.set_ticks(np.arange(0, end, stepsize)) ax.yaxis.set_ticks(np.arange(0, end, stepsize))
x = np.linspace(start, end, limits+1) x = np.linspace(start, end, limits + 1)
y = np.linspace(start, end, limits+1) y = np.linspace(start, end, limits + 1)
ax.fill_between(x, y, end, facecolor='b', alpha=0.3) ax.fill_between(x, y, end, facecolor='b', alpha=0.3)
# plt.plot(np.linspace(0, 1, 10), np.linspace(0, 1, 10), lw=1) # plt.plot(np.linspace(0, 1, 10), np.linspace(0, 1, 10), lw=1)
ax.spines['top'].set_visible(True) ax.spines['top'].set_visible(True)
@@ -744,14 +748,17 @@ def plotScatter(s1,s2,vs,label,limits,type):
tight_bbox=True tight_bbox=True
) )
import threading import threading
class BackgroundTask(object): class BackgroundTask(object):
""" Threading example class """ Threading example class
The run() method will be started and it will run in the background The run() method will be started and it will run in the background
until the application exits. until the application exits.
""" """
def __init__(self, model,PATH, interval=1): def __init__(self, model, PATH, interval=1):
""" Constructor """ Constructor
:type interval: int :type interval: int
:param interval: Check interval, in seconds :param interval: Check interval, in seconds
@@ -761,10 +768,10 @@ class BackgroundTask(object):
self.path = PATH self.path = PATH
thread = threading.Thread(target=self.run, args=()) thread = threading.Thread(target=self.run, args=())
thread.daemon = True # Daemonize thread thread.daemon = True # Daemonize thread
thread.start() # Start the execution thread.start() # Start the execution
def run(self): def run(self):
""" Method that runs forever """ """ Method that runs forever """
self.model.save_model(self.path, self.model.save_model(self.path,
num_iteration=self.model.best_iteration) num_iteration=self.model.best_iteration)
python/javaDS.py
+47 -23
View File
@@ -1,46 +1,70 @@
from pandas import DataFrame
from common.commons import * from common.commons import *
from commitCollector import * from commitCollector import *
from python.settings import * from settings import *
from otherDatasets import markBugFixingPatches from otherDatasets import markBugFixingPatches
DATASET_PATH = REPO_PATH DATASET_PATH = REPO_PATH
DATASET = os.environ["dataset"] DATASET = os.environ["dataset"]
PROJECT_LIST = os.environ["PROJECT_LIST"] PROJECT_LIST = os.environ["PROJECT_LIST"]
def createDS(): def load_commits(repo: str, git_url: str, branch: str) -> DataFrame:
pjList = PROJECT_LIST.split(',') """
Load commits of a repo
:param repo: Repo name (e.g. "fuse")
:param git_url: Git clone url (e.g. "https://github.com/jboss-fuse/fuse.git")
:param branch: Git branch (e.g. "6.3.0.redhat")
:return: Commits DataFrame
"""
commits_pickle = Path(join(COMMIT_DFS, f'{repo}-fix.pickle.gz'))
# Load existing commits
if commits_pickle.is_file():
return pd.read_pickle(commits_pickle)
# Clone new commits
shellCallTemplate('git config --global http.postBuffer 157286400')
shellCallTemplate(f'git -C {DATASET_PATH} clone {git_url}')
logging.info(f'Git repo cloned: {repo}')
commits = getCommitFromRepo(join(REPO_PATH, repo), join(COMMIT_DFS, repo), branch)
commits = markBugFixingPatches(commits, repo)
commits.to_pickle(commits_pickle)
return commits
def createDS(project_list: str = PROJECT_LIST):
"""
:param project_list: Comma-separated list of git project names (projects must exist in dataset.csv)
:return:
"""
pjList: list[str] = project_list.split(',')
# Ensure directories exist
if not os.path.exists(DATASET_PATH): if not os.path.exists(DATASET_PATH):
os.mkdir(DATASET_PATH) os.mkdir(DATASET_PATH)
if not os.path.exists(COMMIT_DFS): if not os.path.exists(COMMIT_DFS):
os.mkdir(COMMIT_DFS) os.mkdir(COMMIT_DFS)
subjects = pd.read_csv(join(ROOT_DIR, 'data', 'dataset.csv')) # Find project repo urls in dataset.csv
subjects: DataFrame = pd.read_csv(join(ROOT_DIR, 'data', 'dataset.csv'))
if pjList == ['ALL']: if pjList == ['ALL']:
tuples = subjects[['Repo', 'GitRepo', 'Branch']].values.tolist() tuples = subjects[['Repo', 'GitRepo', 'Branch']].values.tolist()
else: else:
# repos = subjects.query("Subject == '{0}'".format(subject)).Repo.tolist()
tuples = subjects[subjects.Repo.isin(pjList)][['Repo', 'GitRepo', 'Branch']].values.tolist() tuples = subjects[subjects.Repo.isin(pjList)][['Repo', 'GitRepo', 'Branch']].values.tolist()
for t in tuples: # Loop through repos
repo, src, branch = t for repo, src, branch in tuples:
logging.info(repo) logging.info(f'Processing {repo}')
if isfile(join(COMMIT_DFS, repo + 'Fix.pickle')): commits = load_commits(repo, src, branch)
commits = load_zipped_pickle(join(COMMIT_DFS, repo + 'Fix.pickle'))
else:
cmd = 'git config --global http.postBuffer 157286400'
shellCallTemplate(cmd)
cmd = 'git -C ' + DATASET_PATH + ' clone ' + src
shellCallTemplate(cmd)
logging.info(repo)
getCommitFromRepo(join(REPO_PATH, repo), join(COMMIT_DFS, repo), branch)
rDF = makeDF(join(COMMIT_DFS, repo + '.commits'))
save_zipped_pickle(rDF, join(COMMIT_DFS, repo + ".pickle"))
# return rDF
commits = rDF
commits = markBugFixingPatches(commits, repo)
commits = commits[commits.files.apply(lambda x: np.any([i == 'M' for i in x.values()]))] commits = commits[commits.files.apply(lambda x: np.any([i == 'M' for i in x.values()]))]
# keep only commits that are changing c files (.c) # keep only commits that are changing c files (.c)
commits = commits[commits.files.apply(lambda x: np.all([i.endswith('.java') for i in x.keys()]))] commits = commits[commits.files.apply(lambda x: np.all([i.endswith('.java') for i in x.keys()]))]
python/otherDatasets.py
+82 -81
View File
@@ -1,24 +1,27 @@
from pandas import DataFrame
from common.commons import * from common.commons import *
DATA_PATH = os.environ["DATA_PATH"] DATA_PATH = os.environ["DATA_PATH"]
COMMIT_DFS = os.environ["COMMIT_DFS"] COMMIT_DFS = os.environ["COMMIT_DFS"]
# DATASET_PATH = '/Users/anilkoyuncu/projects/datasets' # DATASET_PATH = '/Users/anilkoyuncu/projects/datasets'
DATASET_PATH = os.environ["REPO_PATH"] DATASET_PATH = Path(os.environ["REPO_PATH"])
DATASET = os.environ["dataset"] DATASET = os.environ["dataset"]
ROOT = os.environ["ROOT_DIR"] ROOT = os.environ["ROOT_DIR"]
PROJECT_LIST = os.environ["PROJECT_LIST"] PROJECT_LIST = os.environ["PROJECT_LIST"]
def filetype_fileter(filename): def filetype_fileter(filename):
# return filename.endswith(u'.java') and not bool(re.search('test.*\/', filename)) # return filename.endswith(u'.java') and not bool(re.search('test.*\/', filename))
return filename.endswith(u'.c') or filename.endswith(u'.h') return filename.endswith(u'.c') or filename.endswith(u'.h')
def checkoutFiles(sha, shaOld, filePath, type, repo=None):
def checkoutFiles(sha,shaOld, filePath,type, repo=None):
try: try:
# folderDiff = join(DATA_PATH, 'gumInput',repoName, 'DiffEntries') # folderDiff = join(DATA_PATH, 'gumInput',repoName, 'DiffEntries')
folderDiff = join(type, 'DiffEntries') folderDiff = join(type, 'DiffEntries')
folderPrev = join(type, 'prevFiles') folderPrev = join(type, 'prevFiles')
folderRev = join( type, 'revFiles') folderRev = join(type, 'revFiles')
if not os.path.exists(folderDiff): if not os.path.exists(folderDiff):
os.mkdir(folderDiff) os.mkdir(folderDiff)
@@ -31,14 +34,13 @@ def checkoutFiles(sha,shaOld, filePath,type, repo=None):
# if repo is None: # if repo is None:
# repo = join(REPO_PATH,repoName) # repo = join(REPO_PATH,repoName)
savePath = filePath.replace('/', '#')
savePath = filePath.replace('/','#')
if not isfile(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath + '.txt'): if not isfile(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath + '.txt'):
cmd = 'git -C ' + repo + ' diff -U ' + shaOld + ':' + filePath + '..' + sha + ':' + filePath # + '> ' + folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt') cmd = 'git -C ' + repo + ' diff -U ' + shaOld + ':' + filePath + '..' + sha + ':' + filePath # + '> ' + folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt')
output,errors = shellGitCheckout(cmd,enc='latin1') output, errors = shellGitCheckout(cmd, enc='latin1')
if errors: if errors:
# print(errors) # print(errors)
raise FileNotFoundError raise FileNotFoundError
@@ -58,31 +60,30 @@ def checkoutFiles(sha,shaOld, filePath,type, repo=None):
'w') as writeFile: 'w') as writeFile:
writeFile.writelines(diffFile) writeFile.writelines(diffFile)
cmd = 'git -C ' + repo + ' show ' + sha + ':' + filePath + '> ' + folderRev + '/' + sha + '_' + shaOld + '_' + savePath
cmd = 'git -C ' + repo + ' show ' + sha + ':' + filePath + '> ' + folderRev + '/' + sha + '_' + shaOld + '_' +savePath
if errors: if errors:
# print(errors) # print(errors)
raise FileNotFoundError raise FileNotFoundError
o,errors= shellGitCheckout(cmd,enc='latin1') o, errors = shellGitCheckout(cmd, enc='latin1')
cmd = 'git -C ' + repo + ' show ' + shaOld + ':' + filePath + '> ' + folderPrev + '/' + 'prev_'+sha + '_' + shaOld + '_' +savePath cmd = 'git -C ' + repo + ' show ' + shaOld + ':' + filePath + '> ' + folderPrev + '/' + 'prev_' + sha + '_' + shaOld + '_' + savePath
if errors: if errors:
# print(errors) # print(errors)
raise FileNotFoundError raise FileNotFoundError
o,errors = shellGitCheckout(cmd,enc='latin1') o, errors = shellGitCheckout(cmd, enc='latin1')
if errors: if errors:
# print(errors) # print(errors)
raise FileNotFoundError raise FileNotFoundError
except FileNotFoundError as fnfe: except FileNotFoundError as fnfe:
if isfile(folderRev + '/' + sha + '_' + shaOld + '_' +savePath): if isfile(folderRev + '/' + sha + '_' + shaOld + '_' + savePath):
os.remove(folderRev + '/' + sha + '_' + shaOld + '_' +savePath) os.remove(folderRev + '/' + sha + '_' + shaOld + '_' + savePath)
if isfile(folderPrev + '/' + 'prev_'+sha + '_' + shaOld + '_' +savePath): if isfile(folderPrev + '/' + 'prev_' + sha + '_' + shaOld + '_' + savePath):
os.remove(folderPrev + '/' + 'prev_'+sha + '_' + shaOld + '_' +savePath) os.remove(folderPrev + '/' + 'prev_' + sha + '_' + shaOld + '_' + savePath)
if isfile(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt')): if isfile(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java', '.txt')):
os.remove(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt')) os.remove(
folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java', '.txt'))
# print(fnfe) # print(fnfe)
# raise Exception(fnfe) # raise Exception(fnfe)
except Exception as e: except Exception as e:
@@ -90,14 +91,14 @@ def checkoutFiles(sha,shaOld, filePath,type, repo=None):
raise Exception(e) raise Exception(e)
def prepareFiles(t,dsName): def prepareFiles(t, dsName):
try: try:
sha,files = t sha, files = t
shaOld = sha + '^' shaOld = sha + '^'
# repo = '/Users/anil.koyuncu/projects/linux' # repo = '/Users/anil.koyuncu/projects/linux'
# repo = join(REPO_PATH,repoName) # repo = join(REPO_PATH,repoName)
gumInputRepo = join(DATASET,dsName) gumInputRepo = join(DATASET, dsName)
if not os.path.exists(join(gumInputRepo)): if not os.path.exists(join(gumInputRepo)):
os.makedirs(gumInputRepo) os.makedirs(gumInputRepo)
@@ -118,35 +119,30 @@ def prepareFiles(t,dsName):
# return # return
nonTest = [] nonTest = []
for k,v in files.items(): for k, v in files.items():
if v == 'M': if v == 'M':
nonTest.append(k) nonTest.append(k)
# if k.endswith('.c') or k.endswith(u'.h'): # if k.endswith('.c') or k.endswith(u'.h'):
# nonTest.append(k) # nonTest.append(k)
# nonTest = [f for f in files.keys() if f.endswith('.c') or f.endswith(u'.h')] # nonTest = [f for f in files.keys() if f.endswith('.c') or f.endswith(u'.h')]
cmd = 'git -C ' + join(DATASET_PATH,dsName) + ' rev-parse --short=6 ' + shaOld out, err = shellGitCheckout(f'git -C {DATASET_PATH / dsName} rev-parse --short=6 {shaOld}', enc='latin1')
shaOld = out.strip()
output, errors = shellGitCheckout(cmd, enc='latin1') cmd = 'git -C ' + join(DATASET_PATH, dsName) + ' rev-parse --short=6 ' + sha
shaOld = output.strip() out, err = shellGitCheckout(cmd, enc='latin1')
sha = out.strip()
cmd = 'git -C ' + join(DATASET_PATH,dsName) + ' rev-parse --short=6 ' + sha
output, errors = shellGitCheckout(cmd, enc='latin1')
sha = output.strip()
if isinstance(nonTest, list): if isinstance(nonTest, list):
for file in nonTest: for file in nonTest:
checkoutFiles(sha,shaOld, file,gumInputRepo,join(DATASET_PATH,dsName)) checkoutFiles(sha, shaOld, file, gumInputRepo, join(DATASET_PATH, dsName))
except Exception as e: except Exception as e:
print(e) print(e)
def checkCommitLog(x,dsName): def checkCommitLog(x, dsName):
# repo = '/Users/anil.koyuncu/projects/linux' cmd = 'git -C ' + join(DATASET_PATH, dsName) + ' show ' + x + " --pretty=\"format:\" --name-status -M100%"
cmd= 'git -C ' + join(DATASET_PATH,dsName) + ' show ' + x + " --pretty=\"format:\" --name-status -M100%"
out, err = shellGitCheckout(cmd, enc='latin1') out, err = shellGitCheckout(cmd, enc='latin1')
log = {} log = {}
@@ -156,19 +152,21 @@ def checkCommitLog(x,dsName):
ftype = line[:1] ftype = line[:1]
log[fname] = ftype log[fname] = ftype
log log
df = pd.DataFrame(data=[[log, x]], columns=['files', 'commit']) df = pd.DataFrame(data=[[log, x]], columns=['files', 'commit'])
return df return df
def getCommitLog(x,dsName):
def getCommitLog(x, dsName):
# repo = '/Users/anil.koyuncu/projects/linux' # repo = '/Users/anil.koyuncu/projects/linux'
# commit, repo = x # commit, repo = x
cmd = 'git -C ' + join(DATASET_PATH,dsName) + '/ ' + " show --pretty=format:'%B' --no-patch " + x cmd = 'git -C ' + join(DATASET_PATH,
dsName) + '/ ' + " show --pretty=format:'%B' --no-patch " + x
output = shellCallTemplate(cmd, 'latin-1') output = shellCallTemplate(cmd, 'latin-1')
# matches = re.finditer(r"\bfix[a-zA-Z]*", output,re.I) # matches = re.finditer(r"\bfix[a-zA-Z]*", output,re.I)
matches = re.finditer(r"\bfix[a-zA-Z]*|\bbug[a-zA-Z]*", output,re.I) matches = re.finditer(r"\bfix[a-zA-Z]*|\bbug[a-zA-Z]*", output, re.I)
match = list(matches) match = list(matches)
fixes = [] fixes = []
if len(match) >= 1: if len(match) >= 1:
@@ -183,32 +181,32 @@ def getCommitLog(x,dsName):
# for m in match: # for m in match:
# links.append(m.group()) # links.append(m.group())
df = pd.DataFrame(data=[[fixes, output,x]], columns=['fixes','log','commit']) df = pd.DataFrame(data=[[fixes, output, x]], columns=['fixes', 'log', 'commit'])
# df = df.T # df = df.T
# df.columns = ['log', 'commit'] # df.columns = ['log', 'commit']
return df return df
output output
def collectBugFixPatches(dsName): def collectBugFixPatches(dsName):
commits = getAllCommits(dsName) commits = getAllCommits(dsName)
# remove commits that are only deleting or adding files # remove commits that are only deleting or adding files
commits = commits[commits.files.apply(lambda x: np.any([i == 'M' for i in x.values()]))] commits = commits[commits.files.apply(lambda x: np.any([i == 'M' for i in x.values()]))]
# keep only commits that are changing c files (.c) # keep only commits that are changing c files (.c)
commits = commits[commits.files.apply(lambda x: np.all([i.endswith('.c') for i in x.keys()]))] commits = commits[commits.files.apply(lambda x: np.all([i.endswith('.c') for i in x.keys()]))]
#not a revert commit # not a revert commit
# commits = commits[~commits.log.apply(lambda x: x.startswith('Revert'))] # commits = commits[~commits.log.apply(lambda x: x.startswith('Revert'))]
# commits = commits[commits.files.apply(lambda x: len(x) == 1)] # commits = commits[commits.files.apply(lambda x: len(x) == 1)]
# commits['cocci'] = commits.log.apply(lambda x: True if re.search('cocci|coccinelle', x) else False) # commits['cocci'] = commits.log.apply(lambda x: True if re.search('cocci|coccinelle', x) else False)
# coccis = commits[commits.cocci].commit.values.tolist() # coccis = commits[commits.cocci].commit.values.tolist()
if dsName == 'linux': if dsName == 'linux':
commits['cocci'] = commits.log.apply(lambda x: True if re.search('cocci|coccinelle', x) else False) commits['cocci'] = commits.log.apply(
lambda x: True if re.search('cocci|coccinelle', x) else False)
fixes = commits[commits.cocci].commit.values.tolist() fixes = commits[commits.cocci].commit.values.tolist()
else: else:
fixes = commits[commits.fixes.str.len()!=0].commit.values.tolist() fixes = commits[commits.fixes.str.len() != 0].commit.values.tolist()
# links = commits[commits.links.str.len()!=0].commit.values.tolist() # links = commits[commits.links.str.len()!=0].commit.values.tolist()
# bugs = set(fixes).union(links).union(coccis) # bugs = set(fixes).union(links).union(coccis)
@@ -217,11 +215,11 @@ def collectBugFixPatches(dsName):
print(len(commits)) print(len(commits))
# for s in a.commit.values.tolist(): # for s in a.commit.values.tolist():
parallelRun(prepareFiles,commits[['commit','files']].values.tolist(),dsName) parallelRun(prepareFiles, commits[['commit', 'files']].values.tolist(), dsName)
# prepareFiles(s) # prepareFiles(s)
def markBugFixingPatches(commits,dsName): def markBugFixingPatches(commits, dsName):
# from pandarallel import pandarallel # from pandarallel import pandarallel
# #
# pandarallel.initialize() # pandarallel.initialize()
@@ -229,8 +227,8 @@ def markBugFixingPatches(commits,dsName):
# commits # commits
f = parallelRunMergeNew(checkCommitLog, commits['commit'].values.tolist(), dsName) f = parallelRunMergeNew(checkCommitLog, commits['commit'].values.tolist(), dsName)
res = pd.merge(commits, f, on=['commit']) res: DataFrame = pd.merge(commits, f, on=['commit'])
commits=res commits = res
# #
# # commits['isC'] = commits.files.apply(lambda x:np.any([i.endswith('.c') or i.endswith('.h') for i in x.keys() ])) # # commits['isC'] = commits.files.apply(lambda x:np.any([i.endswith('.c') or i.endswith('.h') for i in x.keys() ]))
# commits['isC'] = commits.files.apply(lambda x:np.all([i.endswith('.c') for i in x.keys() ])) # commits['isC'] = commits.files.apply(lambda x:np.all([i.endswith('.c') for i in x.keys() ]))
@@ -238,65 +236,65 @@ def markBugFixingPatches(commits,dsName):
# commits = commits[commits.isC == True] # commits = commits[commits.isC == True]
# commits.commit.parallel_apply(getCommitLog) # commits.commit.parallel_apply(getCommitLog)
f = parallelRunMergeNew(getCommitLog, commits['commit'].values.tolist(),dsName) f = parallelRunMergeNew(getCommitLog, commits['commit'].values.tolist(), dsName)
res = pd.merge(commits, f, on=['commit']) res = pd.merge(commits, f, on=['commit'])
save_zipped_pickle(res, join(COMMIT_DFS, dsName+'Fix' + ".pickle"))
return res return res
def getAllCommits(datasetName): def getAllCommits(datasetName):
if isfile(join(COMMIT_DFS,datasetName+'Fix.pickle')): if isfile(join(COMMIT_DFS, datasetName + 'Fix.pickle')):
return load_zipped_pickle(join(COMMIT_DFS,datasetName+'Fix.pickle')) return load_zipped_pickle(join(COMMIT_DFS, datasetName + 'Fix.pickle'))
else: else:
if isfile(join(COMMIT_DFS,datasetName+'.pickle')): if isfile(join(COMMIT_DFS, datasetName + '.pickle')):
commits = load_zipped_pickle(join(COMMIT_DFS,datasetName+'.pickle')) commits = load_zipped_pickle(join(COMMIT_DFS, datasetName + '.pickle'))
else: else:
if not os.path.exists(COMMIT_DFS): if not os.path.exists(COMMIT_DFS):
os.mkdir(COMMIT_DFS) os.mkdir(COMMIT_DFS)
cmd = 'git -C ' + join(DATASET_PATH,
cmd = 'git -C ' + join(DATASET_PATH,datasetName) + " log --no-merges --pretty=format:'{\"commit\":\"%H\",\"commitDate\":\"%ci\",\"title\":\"%f\",\"committer\":\"%ce\"}' > " + join(COMMIT_DFS,datasetName + '.commits') datasetName) + " log --no-merges --pretty=format:'{\"commit\":\"%H\",\"commitDate\":\"%ci\",\"title\":\"%f\",\"committer\":\"%ce\"}' > " + join(
COMMIT_DFS, datasetName + '.commits')
output = shellCallTemplate(cmd, enc='latin1') output = shellCallTemplate(cmd, enc='latin1')
from commitCollector import makeDF from commitCollector import makeDF
rDF = makeDF(join(COMMIT_DFS,datasetName + '.commits')) rDF = makeDF(join(COMMIT_DFS, datasetName + '.commits'))
save_zipped_pickle(rDF, join(COMMIT_DFS, datasetName + ".pickle")) save_zipped_pickle(rDF, join(COMMIT_DFS, datasetName + ".pickle"))
# return rDF # return rDF
commits = rDF commits = rDF
return markBugFixingPatches(commits,datasetName) return markBugFixingPatches(commits, datasetName)
def core(): def core():
datasets = pd.read_csv(join(ROOT,'data', 'datasets.csv')) datasets = pd.read_csv(join(ROOT, 'data', 'datasets.csv'))
# repoList = ['FFmpeg','curl','nginx','openssl','redis','tmux','vlc'] # repoList = ['FFmpeg','curl','nginx','openssl','redis','tmux','vlc']
pjList = PROJECT_LIST.split(',') pjList = PROJECT_LIST.split(',')
if not os.path.exists(DATASET_PATH): if not os.path.exists(DATASET_PATH):
os.mkdir(DATASET_PATH) os.mkdir(DATASET_PATH)
for repo,src in datasets.values.tolist(): for repo, src in datasets.values.tolist():
if(pjList != ['ALL']): if (pjList != ['ALL']):
if repo in pjList: if repo in pjList:
print(repo) print(repo)
cmd = 'git config --global http.postBuffer 157286400' cmd = 'git config --global http.postBuffer 157286400'
shellCallTemplate(cmd) shellCallTemplate(cmd)
cmd = 'git -C ' + DATASET_PATH + ' clone ' + src cmd = 'git -C ' + DATASET_PATH + ' clone ' + src
shellCallTemplate(cmd) shellCallTemplate(cmd)
logging.info(repo) logging.info(repo)
collectBugFixPatches(repo) collectBugFixPatches(repo)
else: else:
cmd = 'git -C ' + DATASET_PATH + ' clone ' + src cmd = 'git -C ' + DATASET_PATH + ' clone ' + src
shellCallTemplate(cmd) shellCallTemplate(cmd)
logging.info(repo) logging.info(repo)
collectBugFixPatches(repo) collectBugFixPatches(repo)
def codeflaws():
cf = listdir(join(DATASET_PATH,'codeflaws'))
type = join(DATASET,'codeflaws') def codeflaws():
cf = listdir(join(DATASET_PATH, 'codeflaws'))
type = join(DATASET, 'codeflaws')
folderDiff = join(type, 'DiffEntries') folderDiff = join(type, 'DiffEntries')
folderPrev = join(type, 'prevFiles') folderPrev = join(type, 'prevFiles')
folderRev = join(type, 'revFiles') folderRev = join(type, 'revFiles')
@@ -308,9 +306,9 @@ def codeflaws():
if not os.path.exists(folderRev): if not os.path.exists(folderRev):
os.makedirs(folderRev) os.makedirs(folderRev)
cfBugs = [i for i in cf if os.path.isdir(join(DATASET_PATH,'codeflaws',i))] cfBugs = [i for i in cf if os.path.isdir(join(DATASET_PATH, 'codeflaws', i))]
for cfBug in cfBugs: for cfBug in cfBugs:
bugs = [i for i in listdir(join(DATASET_PATH,'codeflaws',cfBug)) if i.endswith('.c')] bugs = [i for i in listdir(join(DATASET_PATH, 'codeflaws', cfBug)) if i.endswith('.c')]
bugs.sort() bugs.sort()
if len(bugs) == 2: if len(bugs) == 2:
s1 = bugs[0].replace('.c', '').split('-') s1 = bugs[0].replace('.c', '').split('-')
@@ -318,12 +316,15 @@ def codeflaws():
prev = s1[-1] prev = s1[-1]
rev = s2[-1] rev = s2[-1]
bugName = '-'.join(s1[: -1]) bugName = '-'.join(s1[: -1])
shutil.copy(join(DATASET_PATH,'codeflaws',cfBug,bugs[0]),join(folderPrev,"prev_"+bugName+"-"+prev+"-"+rev+'.c')) shutil.copy(join(DATASET_PATH, 'codeflaws', cfBug, bugs[0]),
shutil.copy(join(DATASET_PATH,'codeflaws',cfBug,bugs[1]),join(folderRev,bugName+"-"+prev+"-"+rev+'.c')) join(folderPrev, "prev_" + bugName + "-" + prev + "-" + rev + '.c'))
cmd = 'diff -u ' + join(DATASET_PATH,'codeflaws',cfBug,bugs[0]) + ' ' + join(DATASET_PATH,'codeflaws',cfBug,bugs[1])+ ' > ' + join(folderDiff,bugName+"-"+prev+"-"+rev+'.c.txt') shutil.copy(join(DATASET_PATH, 'codeflaws', cfBug, bugs[1]),
join(folderRev, bugName + "-" + prev + "-" + rev + '.c'))
cmd = 'diff -u ' + join(DATASET_PATH, 'codeflaws', cfBug, bugs[0]) + ' ' + join(
DATASET_PATH, 'codeflaws', cfBug, bugs[1]) + ' > ' + join(folderDiff,
bugName + "-" + prev + "-" + rev + '.c.txt')
logging.info(cmd) logging.info(cmd)
output, e = shellGitCheckout(cmd) output, e = shellGitCheckout(cmd)
logging.info(output) logging.info(output)
else: else:
print() print()