From 1ff30b9eaf1c479a34d274008c91664f776b23c9 Mon Sep 17 00:00:00 2001 From: anil Date: Sat, 15 Aug 2020 14:35:52 +0200 Subject: [PATCH] some changes in python --- python/getManybugs.py | 12 ++--- python/main.py | 16 +++++-- python/patch_validate.py | 84 +++++++++++++++++++++++----------- python/sprinferIndex.py | 90 +++++++++++++++++++++++++++++++++++++ python/stats.py | 14 ++++-- python/test_patched_file.py | 88 +++++++++++++++++++++++++++++------- python/validate_manybugs.py | 2 +- 7 files changed, 248 insertions(+), 58 deletions(-) diff --git a/python/getManybugs.py b/python/getManybugs.py index 3fdb0ac..2329a4b 100644 --- a/python/getManybugs.py +++ b/python/getManybugs.py @@ -99,8 +99,8 @@ def exportCore(bugName): if os.path.exists(join(BUGDIR, bugName,'diffs')): shutil.rmtree(join(BUGDIR, bugName,'diffs')) # os.makedirs(join(BUGDIR, bugName,'diffs')) - if os.path.exists(join(BUGDIR, bugName, 'src')): - shutil.rmtree(join(BUGDIR, bugName,'src')) + # if os.path.exists(join(BUGDIR, bugName, 'src')): + # shutil.rmtree(join(BUGDIR, bugName,'src')) # os.makedirs(join(BUGDIR, bugName, 'src')) # cmd = 'docker create -ti --name dummy '+output.strip()+' bash' @@ -114,10 +114,10 @@ def exportCore(bugName): - cmd = 'docker cp dummy:/experiment/src '+join(BUGDIR,bugName) - logging.info(cmd) - output, e = shellGitCheckout(cmd) - logging.info(output) + # cmd = 'docker cp dummy:/experiment/src '+join(BUGDIR,bugName) + # logging.info(cmd) + # output, e = shellGitCheckout(cmd) + # logging.info(output) cmd = 'docker cp dummy:/experiment/diffs '+join(BUGDIR,bugName) logging.info(cmd) diff --git a/python/main.py b/python/main.py index fddfed4..8b8d03a 100644 --- a/python/main.py +++ b/python/main.py @@ -33,7 +33,7 @@ if __name__ == '__main__': # subject = 'ALL' # rootType = 'if' - # job = 'patterns' + job = 'indexClusters' print(job) @@ -107,6 +107,12 @@ if __name__ == '__main__': from sprinferIndex import removeDuplicates removeDuplicates() + elif job =='mergeCoccis': + # from sprinferIndex import mergeCoccis + # mergeCoccis() + from sprinferIndex import removeDuplicates2 + removeDuplicates2() + elif job == 'evalManyBugs': # from patchManyBugs import patchCore # patchCore() @@ -176,8 +182,12 @@ if __name__ == '__main__': from stats import defects4jStats defects4jStats() elif job == 'patterns': - from stats import exportAbstractPatterns - exportAbstractPatterns() + + coccis = load_zipped_pickle(join(DATA_PATH, 'allCocciPatterns2.pickle')) + coccis['family'] = coccis.cid.apply(lambda x: x.split('.cocci')[0]) + + # from stats import exportAbstractPatterns + # exportAbstractPatterns() elif job == 'travis': if isfile(join(DATA_PATH,'repoList')): diff --git a/python/patch_validate.py b/python/patch_validate.py index 71cc702..d309038 100755 --- a/python/patch_validate.py +++ b/python/patch_validate.py @@ -4,7 +4,7 @@ import csv import os from common.commons import * DATA_PATH = os.environ["DATA_PATH"] - +ROOT_DIR = os.environ["ROOT_DIR"] def readTestSuite(testPath): regex = r"([p|n0-9]+)\)" with open(testPath,mode='r') as testFile: @@ -127,7 +127,15 @@ def myvalidateCore(t): def validateCore(t): bugName, port = t container = None - with bugzoo.server.ephemeral(port=port, verbose=False, timeout_connection=30000) as client: + cmd = 'bash {} {}'.format(join(ROOT_DIR, 'data', 'startBugzoo.sh'), port) + + with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) as myProcess: + + # o,e = shellGitCheckout(cmd) + url = "http://127.0.0.1:{}".format(port) + timeout_connection = 3000 + client = Client(url, timeout_connection=timeout_connection) + # with bugzoo.server.ephemeral(port=port, verbose=False, timeout_connection=30000) as client: try: bug = client.bugs[bugName] @@ -144,22 +152,22 @@ def validateCore(t): preId, postId = bugName.split(':')[-1].split('-')[-2:] - failing = join(DATA_PATH, 'manybugs', bugName, 'failing.tests.txt') - passing = join(DATA_PATH, 'manybugs', bugName, 'passing.tests.txt') - - if not (isfile(failing)): - return - - cmd = 'docker cp ' +failing+ ' '+container.id +':/experiment/failing.tests.txt ' - o,e = shellGitCheckout(cmd) - cmd = 'sudo chown $(whoami):$(whoami) "{}"' - cmd = cmd.format('failing.tests.txt') - o = client.containers.exec(container=container, command=cmd, context='/experiment/') - cmd = 'docker cp ' +passing+ ' '+container.id +':/experiment/passing.tests.txt ' - o,e = shellGitCheckout(cmd) - cmd = 'sudo chown $(whoami):$(whoami) "{}"' - cmd = cmd.format('passing.tests.txt') - o = client.containers.exec(container=container, command=cmd, context='/experiment/') + # failing = join(DATA_PATH, 'manybugs', bugName, 'failing.tests.txt') + # passing = join(DATA_PATH, 'manybugs', bugName, 'passing.tests.txt') + # + # if not (isfile(failing)): + # return + # + # cmd = 'docker cp ' +failing+ ' '+container.id +':/experiment/failing.tests.txt ' + # o,e = shellGitCheckout(cmd) + # cmd = 'sudo chown $(whoami):$(whoami) "{}"' + # cmd = cmd.format('failing.tests.txt') + # o = client.containers.exec(container=container, command=cmd, context='/experiment/') + # cmd = 'docker cp ' +passing+ ' '+container.id +':/experiment/passing.tests.txt ' + # o,e = shellGitCheckout(cmd) + # cmd = 'sudo chown $(whoami):$(whoami) "{}"' + # cmd = cmd.format('passing.tests.txt') + # o = client.containers.exec(container=container, command=cmd, context='/experiment/') originalBugs = get_filepaths(join(DATA_PATH, 'manybugs', bugName, 'diffs'), preId) for ob in originalBugs: @@ -180,7 +188,25 @@ def validateCore(t): filepath = ob.split('diffs')[-1] cmd = 'cp diffs' + filepath + ' src' + filepath.replace('-' + postId, '') client.containers.exec(container=container, command=cmd, context='/experiment/') - client.containers.build(container) + + times = 0 + name = ob + patch_result = client.containers.build(container) + if patch_result.successful: + failure_cases, failure, total, test_outcomes = test_all(bug, container, client,validTests) + if failure == 0: + times += 1 + output += ' fixed by {}'.format(name) + else: + output += ' {}'.format(failure_cases) + else: + output += ' {}'.format('build error') + output += ' times:{}'.format(times) + if times > 0: + output += ' success' + else: + output += ' failure' + # client.containers.build(container) # diff_files = get_diff(bug.name, client, container) # patch_result = patch_application(client, container, diff_files) # if not patch_result: @@ -191,13 +217,13 @@ def validateCore(t): # # print("Second_test:",end=' ') # post_test_outcomes = {} - post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client,validTests) - output += ','.join(post_failure) + ' ' - if len(post_failure) == 0: - # times += 1 - fix = 'success' - # print("fix {} by {}".format(bugName, patch_name)) - output += 'fix {} '.format(bugName) + # post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client,validTests) + # output += ','.join(post_failure) + ' ' + # if len(post_failure) == 0: + # # times += 1 + # fix = 'success' + # # print("fix {} by {}".format(bugName, patch_name)) + # output += 'fix {} '.format(bugName) # patch_names = os.listdir(join(DATA_PATH,'manybugs',bugName,'patched')) @@ -245,6 +271,10 @@ def validateCore(t): # '' cmd = 'docker stop {}'.format(container.id) out, e = shellGitCheckout(cmd) + try: + client.shutdown() + except Exception as e: + logging.error(e) # docker stop $(docker ps -q) @@ -276,7 +306,7 @@ def patch_validate(): if isinstance(v,logging.Logger): v.setLevel(logging.ERROR) - buglist = listdir(join(DATA_PATH,'manybugs')) + buglist = listdir(join(DATA_PATH,'manybugs_sos')) # for i in range(0,len(bugList)): bugList = [] diff --git a/python/sprinferIndex.py b/python/sprinferIndex.py index e725562..a7232ea 100644 --- a/python/sprinferIndex.py +++ b/python/sprinferIndex.py @@ -5,6 +5,7 @@ SPINFER_PATH = os.environ["spinfer"] SPINFER_INDEX_PATH = os.environ["dataset"] COCCI_PATH = join(os.environ["coccinelle"],'spatch') DATASET = os.environ["dataset"] +ROOT_DIR = os.environ["ROOT_DIR"] def indexCore(): @@ -157,6 +158,43 @@ def runSpinfer(): # output,e = shellGitCheckout(cmd) # logging.info(output) +def mergeCoccis(): + DATA = '/Users/anil.koyuncu/projects/fixminer/fixminer-data2' + folders = os.listdir(DATA) + coccis = [i for i in folders if 'cocci' in i] + + allCoccis = [get_filepaths(join(DATA, i), '') for i in coccis] + + for cocci in list(itertools.chain.from_iterable(allCoccis)): + + _,srcFolder,cocciName = cocci.split(DATA)[-1].split('/') + + with open(cocci, 'r') as iFile: + idx = iFile.readlines() + idx + values = np.array(idx) + points = np.where(values == '@@\n') + + points = list(itertools.chain.from_iterable(points)) + # if len(points) == 0: + # os.remove(join(SPINFER_INDEX_PATH, 'cocci', cocci)) + # patches = list(zip(*([iter(points)] * 2))) + patches = list(pairwise(points[::2]))# every second element in list + + if len(patches) > 0: + i = 0; + for t in patches : + t + with open(join(DATA, 'merged', cocciName+str(i)+'-'+srcFolder), 'w') as iFile: + iFile.writelines(idx[t[0]:t[1]]) + i=i+1 + t + with open(join(DATA, 'merged', cocciName + str(i)+'-'+srcFolder), 'w') as iFile: + iFile.writelines(idx[t[1]:]) + else: + shutil.copy2(cocci,join(DATA, 'merged', cocciName + '-'+srcFolder)) + + def divideCoccis(): import datetime shutil.copytree(join(SPINFER_INDEX_PATH,'cocci'),join(DATA_PATH,'cocci'+ datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))) @@ -200,6 +238,58 @@ def getFreqPatterns(): re.search(r"// Recall:(.*), Precision:(.*), Matching recall:(.*)") +def removeDuplicates2(): + commentPattern = r"(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)" + DATA = '/Users/anil.koyuncu/projects/fixminer/fixminer-data2' + coccis =os.listdir(join(DATA, 'merged')) + cocciPatterns = pd.DataFrame(columns=['cid', 'pattern','inferedFrom','recall','precision','matchingRecall']) + ind = 0 + for cocci in coccis: + try: + if cocci == '.DS_Store': + continue + with open(join(DATA, 'merged', cocci), 'r') as iFile: + idx = iFile.read() + idx + inferedFrom = re.search(r"// Infered from:(.*)\n",idx).groups() + recall,precision, matchingRecall = re.search(r"// Recall:(.*), Precision:(.*), Matching recall:(.*)",idx).groups() + pattern = re.sub(commentPattern, '', idx, re.DOTALL) + cocciPatterns.loc[ind] = [cocci,pattern,inferedFrom,recall.strip(),precision.strip(),matchingRecall.strip()] + ind = ind +1 + except Exception as e: + shutil.move(join(DATA,'merged',cocci),join(DATA,'mergedBroken',cocci)) + continue + # logging.error(e) + cocciPatterns['iFiles'] = cocciPatterns.inferedFrom.apply(lambda x: getInferred(x[0])) + + cocciPatterns['freq'] = cocciPatterns.iFiles.apply(lambda x: len(x)) + cocciPatterns['project'] = cocciPatterns.iFiles.apply(lambda x: list(set([i.split('/{')[0].replace('(','') for i in x]))) + cocciPatterns.sort_values(by='freq', inplace=True, ascending=False) + # save_zipped_pickle(cocciPatterns,join(DATA_PATH,'allCocciPatterns.pickle')) + save_zipped_pickle(cocciPatterns,join(DATA,'allCocciPatternsMerge.pickle')) + + cocciPatterns['dup'] = cocciPatterns.duplicated(subset=['pattern'], keep=False) + + duplicates = cocciPatterns[cocciPatterns.dup == True] + + duplicateDF = duplicates.groupby(by=['pattern'], as_index=False).agg(lambda x: x.tolist()) + + pattern2keep= duplicateDF.cid.apply(lambda x: min(x, key=len)).values.tolist() + allDuplicates = list(itertools.chain.from_iterable(duplicateDF.cid.values.tolist())) + + toRemove = list(set(allDuplicates).difference(set(pattern2keep))) + + for tr in toRemove: + shutil.move(join(DATA,'merged',tr),join(DATA,'mergedDuplicate',tr)) + + # allPatterns = cocciPatterns.cid.values.tolist() + # uniquePatterns = cocciPatterns.drop_duplicates(subset=['pattern']).cid.values.tolist() + # toRemove = list(set(allPatterns).difference(uniquePatterns)) + # print(toRemove) + # for p in toRemove: + # os.remove(join(SPINFER_INDEX_PATH, 'cocci', p)) + # print(len(uniquePatterns)) + def removeDuplicates(): commentPattern = r"(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)" coccis =os.listdir(join(SPINFER_INDEX_PATH, 'cocci')) diff --git a/python/stats.py b/python/stats.py index 62df327..2aa5438 100644 --- a/python/stats.py +++ b/python/stats.py @@ -473,12 +473,18 @@ def exportAbstractPatterns(): if 'java' == PROJECT_TYPE: isJava = True for id, members in df[['cid','members']].values.tolist(): + if (len(members) == 0): + continue + try: + dKey = '/'.join(id[0].split('-')[:-1]) + "/" + members[0] + lines = redis_db.hget("dump",dKey ) - dKey = '/'.join(id[0].split('-')[:-1]) + "/" + members[0] - lines = redis_db.hget("dump",dKey ) - cid = id[0].replace("-",'#') - abstractPattern(cid,lines.decode(),isJava,members) + cid = id[0].replace("-",'#') + + abstractPattern(cid,lines.decode(),isJava,members) + except Exception as e: + logging.error(e) def abstractPattern(cid,lines,isJava,cMembers): diff --git a/python/test_patched_file.py b/python/test_patched_file.py index 50b826a..0d5b946 100755 --- a/python/test_patched_file.py +++ b/python/test_patched_file.py @@ -6,7 +6,34 @@ import signal # from common.commons import shellGitCheckout DATA_PATH = os.environ["DATA_PATH"] ROOT_DIR = os.environ["ROOT_DIR"] +DATASET = os.environ["dataset"] introClassFile = join(DATA_PATH,'introClassData.txt') +COCCI_PATH = join(os.environ["coccinelle"],'spatch') +def patchSourceFile(bugPath,spfile,bugName): + # print(bugPath) + srcName = bugPath.split('/')[-1].split(':')[1] + srcPath = join(bugPath,srcName + '.c') + patchName = srcName+'.c' + + if(isfile(join(DATA_PATH,"introclass",bugName,'patched',patchName+spfile+'.c'))): + return join(DATA_PATH,"introclass",bugName,'patched',patchName+spfile+'.c') + + if not (isfile(join(DATA_PATH,"introclass",bugName,'patches',patchName+spfile+'.txt'))): + cmd = COCCI_PATH + ' --sp-file ' + join(DATASET, 'cocci', spfile) + ' ' + srcPath + ' --patch -o' + join( + DATA_PATH, "introclass", bugName, 'patches', patchName) + ' > ' + join(DATA_PATH, "introclass", bugName, + 'patches', + patchName + spfile + '.txt') + + output, e = shellGitCheckout(cmd) + # logging.info(output) + patchSize = os.path.getsize(join(DATA_PATH,"introclass",bugName,'patches',patchName+spfile+'.txt')) + if patchSize == 0 : + # os.remove(join(DATA_PATH,"introclass",bugName,'patches',patchName+spfile+'.txt')) + return None + else: + cmd = 'patch -d '+join(DATA_PATH, "introclass", bugName)+' -i '+join(DATA_PATH,"introclass",bugName,'patches',patchName+spfile+'.txt')+' -o '+join(DATA_PATH,"introclass",bugName,'patched',patchName+spfile+'.c') + o,e = shellGitCheckout(cmd) + return join(DATA_PATH, "introclass", bugName, 'patched', patchName + spfile + '.c') def testCore(t): bugName, port = t @@ -14,7 +41,7 @@ def testCore(t): # with bugzoo.server.ephemeral(port=port, verbose=False,bugzooPath="/Users/anil.koyuncu/anaconda3/envs/python36/bin/bugzood", timeout_connection=3000) as client: cmd = 'bash {} {}'.format(join(ROOT_DIR,'data','startBugzoo.sh'),port) - with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) as process: + with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) as myProcess: # o,e = shellGitCheckout(cmd) url = "http://127.0.0.1:{}".format(port) @@ -56,27 +83,39 @@ def testCore(t): return '' output += '@fail:' + str(pre_failure) + '@total:' + str(total) + ', ' - # print("patching...") - path = join(DATA_PATH,'introclass',bugName) - patch_path = join(path ,'patched') - # avaliable_patch = os.path.abspath('data') + '/introclass2/' + bugName + '/' + 'patches' - patch_names = os.listdir(patch_path) + spfiles = listdir(join(DATASET, 'cocci')) + # print("patching... " + bugName) + for idx,spfile in enumerate(spfiles): + if spfile == '.DS_Store': + continue - times = 0 - for patch_name in patch_names: + path = join(DATA_PATH,'introclass',bugName) + patch = patchSourceFile(path,spfile,bugName) + + if patch is None: + continue + # print(patch) + # print(str(idx) + '..'+str(len(spfiles))) + # patch_path = join(path ,'patched') + # # avaliable_patch = os.path.abspath('data') + '/introclass2/' + bugName + '/' + 'patches' + # patch_names = os.listdir(patch_path) + # + times = 0 + # for patch_name in patch_names: # if patch_name not in os.listdir(avaliable_patch): # continue - patch = join(patch_path,patch_name) + # patch = join(patch_path,patch_name) patch_result = patched_application(path, bug.name, patch, client, container) if patch_result == -1 or patch_result.code != 0: # print("@{}@".format('False'), end='') # print("{}".format('F'), end=' ') - output += '@False@F ' + # output += '@False@F ' + output += '@False:' + str(idx) + ':' + patch.split('/')[-1] + '@' continue # print("@{}@".format('True'), end='') - output += '@True@' - + # output += '@True@' + output += '@True:' + str(idx) + ':' + patch.split('/')[-1] + '@' # print("Second_test:", end=' ') post_test_outcomes = {} post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client) @@ -86,7 +125,8 @@ def testCore(t): times += 1 fix = 'success' # print("fix {} by {}".format(bugName, patch_name)) - output += 'fix {} by {} '.format(bugName, patch_name) + output += 'fix {} by {} '.format(bugName, patch) + break # print("@fail:{}@total:{}".format(post_failure, total),end=' ') # print("@post_failure_cases:{}".format(post_failure_cases)) @@ -103,8 +143,12 @@ def testCore(t): # '' cmd = 'docker stop {}'.format(container.id) out, e = shellGitCheckout(cmd) - client.shutdown() - os.killpg(process.pid, signal.SIGTERM) + try: + client.shutdown() + except Exception as e: + logging.error(e) + # print(myProcess.pid) + # os.killpg(myProcess.pid, signal.SIGTERM) # docker stop $(docker ps -q) @@ -124,6 +168,10 @@ def patch_validate(): # bug = client.bugs['introclass:checksum:08c7ea:006'] bugList = [] + nonFail = ['introclass:syllables:93f87b:005','introclass:smallest:f94e26:004','introclass:smallest:d25c71:001','introclass:median:d25c71:002','introclass:smallest:d25c71:000' + ,'introclass:median:d6364e:007','introclass:median:489253:007','introclass:syllables:d12048:004','introclass:smallest:d9e7ea:002','introclass:syllables:035fe9:000' + ,'introclass:syllables:c9d718:002','introclass:syllables:ea67b8:007','introclass:median:48b829:000','introclass:syllables:d9e7ea:001'] + # cmd = 'bash ' + join(DATA_PATH,'startBugzoo.sh') # cmd = "/Users/anil.koyuncu/anaconda3/envs/python36/bin/bugzood --debug -p " + str(port) # output, errors = shellGitCheckout(cmd) @@ -131,6 +179,11 @@ def patch_validate(): port = 6000 bugs2test= listdir(join(DATA_PATH, "introclass")) for b in bugs2test: + if b == '.DS_Store': + continue + + if b in nonFail: + continue t = b, port bugList.append(t) if port == 6300: @@ -149,8 +202,9 @@ def patch_validate(): # t = 'introclass:syllables:93f87b:005',6000 # t = 'introclass:syllables:99cbb4:000',6000 # testCore(t) - results = parallelRunMerge(testCore, bugList,max_workers=10) - # print('\n'.join(results)) + # results = parallelRunMerge(testCore, bugList,max_workers=10) + results = parallelRunMerge(testCore, bugList) + print('\n'.join(results)) with open(join(DATA_PATH, 'introTestResults'), 'w', encoding='utf-8') as writeFile: # if levelPatch == 0: diff --git a/python/validate_manybugs.py b/python/validate_manybugs.py index 1358179..2d06ef7 100644 --- a/python/validate_manybugs.py +++ b/python/validate_manybugs.py @@ -75,7 +75,7 @@ def patch_validate(t): finally: cmd = 'docker stop {}'.format(container.id) out, e = shellGitCheckout(cmd) - client.shutdown() + # client.shutdown() os.killpg(process.pid, signal.SIGTERM) from bugzoo import Patch, Client