[O] Reformat, optimize, add comments

python/commitCollector.py

@@ -1,4 +1,6 @@
+import json
 
+import pandas as pd
 
 from common.commons import *
 
@@ -7,30 +9,37 @@ DATA_PATH = os.environ["DATA_PATH"]
 COMMIT_DFS = os.environ["COMMIT_DFS"]
 COMMIT_FOLDER = os.environ["COMMIT_FOLDER"]
 
-def getCommitFromRepo(f,gitrepo,branch):
-    cmd = 'git -C ' + f + ' checkout -f ' + branch
 
-    output, err = shellGitCheckout(cmd)
+def getCommitFromRepo(f: PathLike, gitrepo: str, branch: str):
+    """
+
+    :param f: Git repo directory
+    :param gitrepo: Repo name
+    :param branch: Branch name
+    :return: None
+    """
+    file = f'{gitrepo}.commits'
+    output, err = shellGitCheckout(f'git -C {f} checkout -f {branch}')
     m = re.search(branch, err)
 
     while not m:
         time.sleep(10)
         logging.info('Waiting for checkout')
-    cmd = 'git -C ' + f + " log --no-merges --pretty=format:'{\"commit\":\"%H\",\"commitDate\":\"%ci\",\"title\":\"%f\",\"committer\":\"%ce\"}' > " + gitrepo + '.commits'
-    output = shellCallTemplate(cmd,enc='latin1')
 
+    # Create commits file
+    form = json.dumps({"commit": "%H", "commitDate": "%ci", "title": "%f", "committer": "%ce"})
+    shellCallTemplate(f"git -C {f} log --no-merges --pretty=format:'{form}' > {file}", enc='latin1')
 
-def makeDF(filename):
+    # Collect commits
-    with open(filename,encoding='latin1') as f:
+    commits = json.loads(f'[{Path(file).read_text()}]')
-        lines = f.readlines()
+
-    ls = [eval(f) for f in lines]
+    # Convert to DataFrame
-    ds  = pd.DataFrame.from_dict(ls)
+    ds = pd.DataFrame.from_dict(commits)
-    ds['commitDate']= ds['commitDate'].apply(lambda x:pd.to_datetime(x))
+    ds['commitDate'] = pd.to_datetime(ds['commitDate'])
     return ds
 
 
 def caseCollect(subject):
-    
     if not os.path.exists(COMMIT_FOLDER):
         os.mkdir(COMMIT_FOLDER)
     if not os.path.exists(COMMIT_DFS):
@@ -41,7 +50,8 @@ def caseCollect(subject):
         tuples = subjects[['Repo', 'Branch']].values.tolist()
     else:
         # repos = subjects.query("Subject == '{0}'".format(subject)).Repo.tolist()
-        tuples = subjects.query("Subject == '{0}'".format(subject))[['Repo', 'Branch']].values.tolist()
+        tuples = subjects.query("Subject == '{0}'".format(subject))[
+            ['Repo', 'Branch']].values.tolist()
 
     for t in tuples:
         repo, branch = t
@@ -60,6 +70,7 @@ def caseCollect(subject):
         save_zipped_pickle(rDF, join(COMMIT_DFS, repoName + ".pickle"))
         # p.dump(rDF, open(join(COMMIT_DFS, repoName + ".pickle"), "wb"))
 
+
 def caseClone(subject):
     if not os.path.exists(REPO_PATH):
         os.mkdir(REPO_PATH)
@@ -74,4 +85,3 @@ def caseClone(subject):
     for gitrepo in gitrepos:
         cmd = 'git clone ' + gitrepo
         out = shellCallTemplate(cmd)
-

python/common/commons.py

@@ -1,7 +1,8 @@
-
 import logging
 import sys
 import gzip
+from typing import Union
+
 import numpy as np
 from tqdm import tqdm
 import shutil
@@ -28,16 +29,18 @@ import datetime
 import subprocess
 from pathlib import Path
 
-
+PathLike = Union[os.PathLike, str]
 
 sourceCodeColumns = ['packageName', 'className', 'methodNames', 'formalParameter',
-                       'methodInvocation', 'memberReference', 'documentation', 'literal', 'rawSource', 'hunks',
+                     'methodInvocation', 'memberReference', 'documentation', 'literal', 'rawSource',
+                     'hunks',
                      'commitLogs', 'classNameExt']
 
 
 def nap():
     time.sleep(1)
 
+
 def setLogg():
     # logging.basicConfig(filename='app.log', filemode='w',level=logging.DEBUG)
     root = logging.getLogger()
@@ -45,7 +48,8 @@ def setLogg():
 
     ch = logging.StreamHandler(sys.stdout)
     ch.setLevel(logging.WARNING)
-    formatter = logging.Formatter('%(asctime)s - %(process)d - %(levelname)s - %(filename)s:%(funcName)s - %(message)s')
+    formatter = logging.Formatter(
+        '%(asctime)s - %(process)d - %(levelname)s - %(filename)s:%(funcName)s - %(message)s')
     ch.setFormatter(formatter)
     # ch.addFilter(lambda record: record.levelno <= logging.)
     root.addHandler(ch)
@@ -64,6 +68,7 @@ def setLogg():
     h2.setFormatter(formatter)
     root.addHandler(h2)
 
+
 def setEnv(args):
     # env = args.env
 
@@ -72,7 +77,6 @@ def setEnv(args):
     os.environ["ROOT_DIR"] = args.root
     sys.path.append(args.root)
 
-
     import yaml
     # if os.uname().nodename != '':
     #     with open(join(os.environ["ROOT_DIR"], os.uname().nodename + ".config.yml"), 'r') as ymlfile:
@@ -113,7 +117,6 @@ def setEnv(args):
     # os.environ["JDK8"] = cfg['java']['8home']
     # os.environ["D4JHOME"] = cfg['defects4j']['home']
 
-
     os.environ["CODE_PATH"] = join(os.environ["ROOT_DIR"], 'code/')
     # os.environ["DATA_PATH"] = join(os.environ["ROOT_DIR"],'data/')
     # os.environ["REPO_PATH"] = join(os.environ["DATA_PATH"], 'gitrepo/')
@@ -139,10 +142,6 @@ def setEnv(args):
     os.environ["DATASET_DIR"] = join(os.environ["DATA_PATH"], 'datasets/')
     os.environ["REMOTE_PATH"] = '/Volumes/Samsung_T5/data'
 
-
-
-
-
     logging.info('ROOT_DIR : %s', os.environ["ROOT_DIR"])
     logging.info('REPO_PATH : %s', os.environ["REPO_PATH"])
     logging.info('CODE_PATH : %s', os.environ["CODE_PATH"])
@@ -159,7 +158,6 @@ def setEnv(args):
     logging.info('DATASET_DIR : %s', os.environ["DATASET_DIR"])
 
 
-
 def getRun():
     import argparse
     parser = argparse.ArgumentParser(description='')
@@ -168,7 +166,6 @@ def getRun():
     parser.add_argument('-job', dest='job', help='job name')
     parser.add_argument('-prop', dest='prop', help='property file')
 
-
     args = parser.parse_args()
 
     if args.root is None or args.job is None or args.prop is None:
@@ -177,7 +174,6 @@ def getRun():
     return args
 
 
-
 def shellCallTemplate4jar(cmd, enc='utf-8'):
     process = subprocess.Popen(cmd,
                                stdout=subprocess.PIPE, stderr=PIPE, shell=True, encoding=enc,
@@ -195,6 +191,7 @@ def shellCallTemplate4jar(cmd,enc='utf-8'):
                 print(output.strip())
             break
 
+
 def shellCallTemplate(cmd, enc='utf-8'):
     try:
         logging.info(cmd)
@@ -212,8 +209,8 @@ def shellCallTemplate(cmd,enc='utf-8'):
         logging.error(e)
     return output
 
-def getChildMem(pid,children):
 
+def getChildMem(pid, children):
     out = subprocess.Popen(['pgrep', '-P', str(pid)],
                            stdout=subprocess.PIPE).communicate()[0].split(b'\n')
     child = out[0].decode()
@@ -223,8 +220,8 @@ def getChildMem(pid,children):
     else:
         return children
 
-def getAllChildMe(pid):
 
+def getAllChildMe(pid):
     childrenProcess = []
     getChildMem(pid, childrenProcess)
 
@@ -232,6 +229,8 @@ def getAllChildMe(pid):
     return sum(map(memory_usage_ps, childrenProcess)) + memory_usage_ps(pid)
     # else:
     #     return memory_usage_ps(child) + memory_usage_ps(pid)
+
+
 def memory_usage_ps(pid):
     import subprocess
     out = subprocess.Popen(['ps', 'v', '-p', str(pid)],
@@ -243,13 +242,14 @@ def memory_usage_ps(pid):
         mem = float(0)
     return mem
 
+
 def raiseTime(cmd, timeout, my_timer):
     my_timer.cancel()
     raise TimeoutExpired(cmd, timeout)
 
+
 def killP(pid):
-    out = subprocess.Popen(['kill', str(pid)],
+    out = subprocess.Popen(['kill', str(pid)], stdout=subprocess.PIPE).communicate()[0].split(b'\n')
-    stdout=subprocess.PIPE).communicate()[0].split(b'\n')
     out
 
 
@@ -276,6 +276,7 @@ def shellGitCheckout(cmd,timeout =600,enc='utf-8'):
             logging.warning(cmd + '\t' + str(t))
     return output, errors
 
+
 def callSpinfer(cmd, timeout=600, enc='utf-8'):
     output = ''
     errors = ''
@@ -309,7 +310,6 @@ def callSpinfer(cmd,timeout =600,enc='utf-8'):
         except TimeoutExpired as t:
             # my_timer.cancel()
 
-
             childrenProcess = []
             getChildMem(p.pid, childrenProcess)
             [killP(i) for i in childrenProcess]
@@ -320,20 +320,24 @@ def callSpinfer(cmd,timeout =600,enc='utf-8'):
             logging.warning(cmd + '\t' + str(t))
     return output, errors
 
+
 def save_zipped_pickle(obj, filename, protocol=-1):
     with gzip.open(filename, 'wb') as f:
         p.dump(obj, f, protocol)
 
+
 def load_zipped_pickle(filename):
     with gzip.open(filename, 'rb') as f:
         loaded_object = p.load(f)
         return loaded_object
 
+
 def file2path(file):
     count = file.count(".") - 1
     file = file.replace('.', '/', count)
     return file
 
+
 def isFileInList(file, checkList):
     for f in checkList:
         if f in file:
@@ -341,6 +345,7 @@ def isFileInList(file,checkList):
     return False
     # [i for i in ansFiles if 'org/fusesource/esb/itests/basic/fabric/EsbFeatureTest.java' in i]
 
+
 def get_venn_sections(sets):
     """
     Given a list of sets, return a new list of sets with all the possible
@@ -378,12 +383,14 @@ def get_venn_sections(sets):
         combo_sets[tag] = combo
     return combo_sets
 
+
 def pairwise(iterable):
     "s -> (s0,s1), (s1,s2), (s2, s3), ..."
     a, b = itertools.tee(iterable)
     next(b, None)
     return zip(a, b)
 
+
 def RR_XGB(x, ao, column):
     if x[ao] == 1:
         return (1.0 / (x[column]))
@@ -392,6 +399,7 @@ def RR_XGB(x,ao,column):
     else:
         return 0
 
+
 def parallelRunNo(coreFun, elements, *args):
     with concurrent.futures.ProcessPoolExecutor(max_workers=int(8)) as executor:
         try:
@@ -474,7 +482,6 @@ def parallelRunMerge(coreFun,elements,*args,max_workers=os.cpu_count()):
 
 
 def parallelRunMergeNew(coreFun, elements, *args, max_workers=os.cpu_count()):
-
     res = []
     with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
         try:
@@ -503,8 +510,8 @@ def parallelRunMergeNew(coreFun,elements,*args,max_workers=os.cpu_count()):
     aDF = pd.concat(res)
     return aDF
 
-def get_filepaths(directory,extension):
 
+def get_filepaths(directory, extension):
     file_paths = []  # List which will store all of the full filepaths.\n,
     exclude = '.git'
     # Walk the tree.\n,
@@ -520,6 +527,7 @@ def get_filepaths(directory,extension):
 
     return file_paths  # Self-explanatory.\n,
 
+
 def get_class_weights(y):
     counter = Counter(y)
     majority = max(counter.values())
@@ -533,7 +541,6 @@ def stopDB(dbDir,portInner):
     logging.info(o)
 
 
-
 def startDB(dbDir, portInner, projectType):
     dbName = "dumps-" + projectType + ".rdb"
     # portInner = '6380'
@@ -569,13 +576,12 @@ def unique_everseen(iterable, key=None):
                 seen_add(k)
                 yield element
 
+
 def plotBox(yList, labels, fn, rotate=False, limit=True):
     import matplotlib
     matplotlib.use("TkAgg")
     import matplotlib.pyplot as plt
 
-
-
     fig = plt.figure()
     ax1 = fig.add_subplot(111)
     meanpointsprops = dict(markeredgecolor='blue', markerfacecolor=
@@ -583,7 +589,8 @@ def plotBox(yList,labels, fn, rotate=False,limit=True):
 
     flierprops = dict(markeredgecolor='black', markerfacecolor=
     'black', marker='.', markersize=2)
-    box = ax1.boxplot(yList, 0, flierprops=flierprops,widths=0.5, showmeans=False, vert=True,meanprops=meanpointsprops)
+    box = ax1.boxplot(yList, 0, flierprops=flierprops, widths=0.5, showmeans=False, vert=True,
+                      meanprops=meanpointsprops)
     for line in box['medians']:
         x, y = line.get_xydata()[1]
         line.set(linewidth=3)
@@ -616,20 +623,18 @@ def plotBox(yList,labels, fn, rotate=False,limit=True):
     fig.set_size_inches(7, 1, forward=True)
     fig.savefig(fn, dpi=100, bbox_inches='tight')
 
-
     plt.show()
 
+
 def plotBox2(ys, labels, fn, means, rotate=False, limit=True):
-
-
     import matplotlib
     matplotlib.use("TkAgg")
     import matplotlib.pyplot as plt
 
-
     fig, axes = plt.subplots(nrows=3, ncols=1)
 
-    for ax1,yList,l,l2,mean in zip(axes.flat,ys,labels,['Shapes','Actions','Tokens'],means):
+    for ax1, yList, l, l2, mean in zip(axes.flat, ys, labels, ['Shapes', 'Actions', 'Tokens'],
+                                       means):
         # plt.setp(ax1.get_xticks(),visible=False)
         # ax1 = fig.add_subplot(111)
         meanpointsprops = dict(markeredgecolor='blue', markerfacecolor=
@@ -637,7 +642,8 @@ def plotBox2(ys,labels, fn,means, rotate=False,limit=True):
 
         flierprops = dict(markeredgecolor='black', markerfacecolor=
         'black', marker='.', markersize=2)
-        box = ax1.boxplot(yList, 0, flierprops=flierprops,widths=0.5, showmeans=False, vert=True,meanprops=meanpointsprops)
+        box = ax1.boxplot(yList, 0, flierprops=flierprops, widths=0.5, showmeans=False, vert=True,
+                          meanprops=meanpointsprops)
 
         ax1.axhline(linewidth=2, color='r', y=mean)
 
@@ -687,12 +693,10 @@ def plotBox2(ys,labels, fn,means, rotate=False,limit=True):
     plt.subplots_adjust(wspace=0, hspace=0.05)
     fig = plt.gcf()
 
-
     # fig.tight_layout()
     fig.set_size_inches(7, 7, forward=True)
     fig.savefig(fn, dpi=100, bbox_inches='tight')
 
-
     plt.show()
 
 
@@ -744,7 +748,10 @@ def plotScatter(s1,s2,vs,label,limits,type):
         tight_bbox=True
     )
 
+
 import threading
+
+
 class BackgroundTask(object):
     """ Threading example class
     The run() method will be started and it will run in the background

python/javaDS.py

@@ -1,46 +1,70 @@
+from pandas import DataFrame
+
 from common.commons import *
 from commitCollector import *
-from python.settings import *
+from settings import *
 
 from otherDatasets import markBugFixingPatches
 
+
 DATASET_PATH = REPO_PATH
 DATASET = os.environ["dataset"]
 PROJECT_LIST = os.environ["PROJECT_LIST"]
 
 
-def createDS():
+def load_commits(repo: str, git_url: str, branch: str) -> DataFrame:
-    pjList = PROJECT_LIST.split(',')
+    """
+    Load commits of a repo
+
+    :param repo: Repo name (e.g. "fuse")
+    :param git_url: Git clone url (e.g. "https://github.com/jboss-fuse/fuse.git")
+    :param branch: Git branch (e.g. "6.3.0.redhat")
+    :return: Commits DataFrame
+    """
+    commits_pickle = Path(join(COMMIT_DFS, f'{repo}-fix.pickle.gz'))
+
+    # Load existing commits
+    if commits_pickle.is_file():
+        return pd.read_pickle(commits_pickle)
+
+    # Clone new commits
+    shellCallTemplate('git config --global http.postBuffer 157286400')
+    shellCallTemplate(f'git -C {DATASET_PATH} clone {git_url}')
+    logging.info(f'Git repo cloned: {repo}')
+
+    commits = getCommitFromRepo(join(REPO_PATH, repo), join(COMMIT_DFS, repo), branch)
+    commits = markBugFixingPatches(commits, repo)
+    commits.to_pickle(commits_pickle)
+
+    return commits
+
+
+def createDS(project_list: str = PROJECT_LIST):
+    """
+
+    :param project_list: Comma-separated list of git project names (projects must exist in dataset.csv)
+    :return:
+    """
+    pjList: list[str] = project_list.split(',')
+
+    # Ensure directories exist
     if not os.path.exists(DATASET_PATH):
         os.mkdir(DATASET_PATH)
     if not os.path.exists(COMMIT_DFS):
         os.mkdir(COMMIT_DFS)
 
-    subjects = pd.read_csv(join(ROOT_DIR, 'data', 'dataset.csv'))
+    # Find project repo urls in dataset.csv
-
+    subjects: DataFrame = pd.read_csv(join(ROOT_DIR, 'data', 'dataset.csv'))
     if pjList == ['ALL']:
         tuples = subjects[['Repo', 'GitRepo', 'Branch']].values.tolist()
     else:
-        # repos = subjects.query("Subject == '{0}'".format(subject)).Repo.tolist()
         tuples = subjects[subjects.Repo.isin(pjList)][['Repo', 'GitRepo', 'Branch']].values.tolist()
 
-    for t in tuples:
+    # Loop through repos
-        repo, src, branch = t
+    for repo, src, branch in tuples:
-        logging.info(repo)
+        logging.info(f'Processing {repo}')
-        if isfile(join(COMMIT_DFS, repo + 'Fix.pickle')):
+        commits = load_commits(repo, src, branch)
-            commits = load_zipped_pickle(join(COMMIT_DFS, repo + 'Fix.pickle'))
+
-        else:
-            cmd = 'git config --global http.postBuffer 157286400'
-            shellCallTemplate(cmd)
-            cmd = 'git -C ' + DATASET_PATH + ' clone ' + src
-            shellCallTemplate(cmd)
-            logging.info(repo)
-            getCommitFromRepo(join(REPO_PATH, repo), join(COMMIT_DFS, repo), branch)
-            rDF = makeDF(join(COMMIT_DFS, repo + '.commits'))
-            save_zipped_pickle(rDF, join(COMMIT_DFS, repo + ".pickle"))
-            # return rDF
-            commits = rDF
-            commits = markBugFixingPatches(commits, repo)
         commits = commits[commits.files.apply(lambda x: np.any([i == 'M' for i in x.values()]))]
         # keep only commits that are changing c files (.c)
         commits = commits[commits.files.apply(lambda x: np.all([i.endswith('.java') for i in x.keys()]))]

python/otherDatasets.py

@@ -1,18 +1,21 @@
+from pandas import DataFrame
+
 from common.commons import *
+
 DATA_PATH = os.environ["DATA_PATH"]
 COMMIT_DFS = os.environ["COMMIT_DFS"]
 # DATASET_PATH = '/Users/anilkoyuncu/projects/datasets'
-DATASET_PATH = os.environ["REPO_PATH"]
+DATASET_PATH = Path(os.environ["REPO_PATH"])
 DATASET = os.environ["dataset"]
 ROOT = os.environ["ROOT_DIR"]
 PROJECT_LIST = os.environ["PROJECT_LIST"]
 
+
 def filetype_fileter(filename):
     # return filename.endswith(u'.java') and not bool(re.search('test.*\/', filename))
     return filename.endswith(u'.c') or filename.endswith(u'.h')
 
 
-
 def checkoutFiles(sha, shaOld, filePath, type, repo=None):
     try:
         # folderDiff = join(DATA_PATH, 'gumInput',repoName, 'DiffEntries')
@@ -31,7 +34,6 @@ def checkoutFiles(sha,shaOld, filePath,type, repo=None):
         # if repo is None:
         #     repo = join(REPO_PATH,repoName)
 
-
         savePath = filePath.replace('/', '#')
 
         if not isfile(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath + '.txt'):
@@ -58,8 +60,6 @@ def checkoutFiles(sha,shaOld, filePath,type, repo=None):
                       'w') as writeFile:
                 writeFile.writelines(diffFile)
 
-
-
             cmd = 'git -C ' + repo + ' show ' + sha + ':' + filePath + '> ' + folderRev + '/' + sha + '_' + shaOld + '_' + savePath
 
             if errors:
@@ -82,7 +82,8 @@ def checkoutFiles(sha,shaOld, filePath,type, repo=None):
         if isfile(folderPrev + '/' + 'prev_' + sha + '_' + shaOld + '_' + savePath):
             os.remove(folderPrev + '/' + 'prev_' + sha + '_' + shaOld + '_' + savePath)
         if isfile(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java', '.txt')):
-            os.remove(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt'))
+            os.remove(
+                folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java', '.txt'))
         # print(fnfe)
         # raise Exception(fnfe)
     except Exception as e:
@@ -125,27 +126,22 @@ def prepareFiles(t,dsName):
                 #    nonTest.append(k)
         # nonTest = [f for f in files.keys() if f.endswith('.c') or f.endswith(u'.h')]
 
-        cmd = 'git -C ' + join(DATASET_PATH,dsName) + ' rev-parse --short=6 ' + shaOld
+        out, err = shellGitCheckout(f'git -C {DATASET_PATH / dsName} rev-parse --short=6 {shaOld}', enc='latin1')
-
+        shaOld = out.strip()
-        output, errors = shellGitCheckout(cmd, enc='latin1')
-        shaOld = output.strip()
 
         cmd = 'git -C ' + join(DATASET_PATH, dsName) + ' rev-parse --short=6 ' + sha
-        output, errors = shellGitCheckout(cmd, enc='latin1')
+        out, err = shellGitCheckout(cmd, enc='latin1')
-        sha = output.strip()
+        sha = out.strip()
 
         if isinstance(nonTest, list):
             for file in nonTest:
                 checkoutFiles(sha, shaOld, file, gumInputRepo, join(DATASET_PATH, dsName))
 
-
-
     except Exception as e:
         print(e)
 
 
 def checkCommitLog(x, dsName):
-    # repo = '/Users/anil.koyuncu/projects/linux'
     cmd = 'git -C ' + join(DATASET_PATH, dsName) + ' show ' + x + " --pretty=\"format:\" --name-status -M100%"
 
     out, err = shellGitCheckout(cmd, enc='latin1')
@@ -159,11 +155,13 @@ def checkCommitLog(x,dsName):
     df = pd.DataFrame(data=[[log, x]], columns=['files', 'commit'])
     return df
 
+
 def getCommitLog(x, dsName):
     # repo = '/Users/anil.koyuncu/projects/linux'
     # commit, repo = x
 
-    cmd = 'git -C ' + join(DATASET_PATH,dsName) + '/ ' + " show --pretty=format:'%B' --no-patch " + x
+    cmd = 'git -C ' + join(DATASET_PATH,
+                           dsName) + '/ ' + " show --pretty=format:'%B' --no-patch " + x
 
     output = shellCallTemplate(cmd, 'latin-1')
 
@@ -189,10 +187,9 @@ def getCommitLog(x,dsName):
 
     return df
 
-
-
     output
 
+
 def collectBugFixPatches(dsName):
     commits = getAllCommits(dsName)
     # remove commits that are only deleting or adding files
@@ -205,7 +202,8 @@ def collectBugFixPatches(dsName):
     # commits['cocci'] = commits.log.apply(lambda x: True if re.search('cocci|coccinelle', x) else False)
     # coccis = commits[commits.cocci].commit.values.tolist()
     if dsName == 'linux':
-        commits['cocci'] = commits.log.apply(lambda x: True if re.search('cocci|coccinelle', x) else False)
+        commits['cocci'] = commits.log.apply(
+            lambda x: True if re.search('cocci|coccinelle', x) else False)
         fixes = commits[commits.cocci].commit.values.tolist()
     else:
         fixes = commits[commits.fixes.str.len() != 0].commit.values.tolist()
@@ -229,7 +227,7 @@ def markBugFixingPatches(commits,dsName):
     # commits
 
     f = parallelRunMergeNew(checkCommitLog, commits['commit'].values.tolist(), dsName)
-    res = pd.merge(commits, f, on=['commit'])
+    res: DataFrame = pd.merge(commits, f, on=['commit'])
     commits = res
     #
     # # commits['isC'] = commits.files.apply(lambda x:np.any([i.endswith('.c') or i.endswith('.h') for i in x.keys() ]))
@@ -241,8 +239,6 @@ def markBugFixingPatches(commits,dsName):
     f = parallelRunMergeNew(getCommitLog, commits['commit'].values.tolist(), dsName)
 
     res = pd.merge(commits, f, on=['commit'])
-
-    save_zipped_pickle(res, join(COMMIT_DFS, dsName+'Fix' + ".pickle"))
     return res
 
 
@@ -257,8 +253,9 @@ def getAllCommits(datasetName):
             if not os.path.exists(COMMIT_DFS):
                 os.mkdir(COMMIT_DFS)
 
-
+            cmd = 'git -C ' + join(DATASET_PATH,
-            cmd = 'git -C ' + join(DATASET_PATH,datasetName) + " log --no-merges --pretty=format:'{\"commit\":\"%H\",\"commitDate\":\"%ci\",\"title\":\"%f\",\"committer\":\"%ce\"}' > " + join(COMMIT_DFS,datasetName + '.commits')
+                                   datasetName) + " log --no-merges --pretty=format:'{\"commit\":\"%H\",\"commitDate\":\"%ci\",\"title\":\"%f\",\"committer\":\"%ce\"}' > " + join(
+                COMMIT_DFS, datasetName + '.commits')
             output = shellCallTemplate(cmd, enc='latin1')
 
             from commitCollector import makeDF
@@ -293,6 +290,7 @@ def core():
             logging.info(repo)
             collectBugFixPatches(repo)
 
+
 def codeflaws():
     cf = listdir(join(DATASET_PATH, 'codeflaws'))
 
@@ -318,12 +316,15 @@ def codeflaws():
             prev = s1[-1]
             rev = s2[-1]
             bugName = '-'.join(s1[: -1])
-            shutil.copy(join(DATASET_PATH,'codeflaws',cfBug,bugs[0]),join(folderPrev,"prev_"+bugName+"-"+prev+"-"+rev+'.c'))
+            shutil.copy(join(DATASET_PATH, 'codeflaws', cfBug, bugs[0]),
-            shutil.copy(join(DATASET_PATH,'codeflaws',cfBug,bugs[1]),join(folderRev,bugName+"-"+prev+"-"+rev+'.c'))
+                        join(folderPrev, "prev_" + bugName + "-" + prev + "-" + rev + '.c'))
-            cmd = 'diff -u ' + join(DATASET_PATH,'codeflaws',cfBug,bugs[0]) + ' ' + join(DATASET_PATH,'codeflaws',cfBug,bugs[1])+ ' > ' + join(folderDiff,bugName+"-"+prev+"-"+rev+'.c.txt')
+            shutil.copy(join(DATASET_PATH, 'codeflaws', cfBug, bugs[1]),
+                        join(folderRev, bugName + "-" + prev + "-" + rev + '.c'))
+            cmd = 'diff -u ' + join(DATASET_PATH, 'codeflaws', cfBug, bugs[0]) + ' ' + join(
+                DATASET_PATH, 'codeflaws', cfBug, bugs[1]) + ' > ' + join(folderDiff,
+                                                                          bugName + "-" + prev + "-" + rev + '.c.txt')
             logging.info(cmd)
             output, e = shellGitCheckout(cmd)
             logging.info(output)
         else:
             print()
-