some changes in python

This commit is contained in:
anil
2020-08-15 14:35:52 +02:00
parent c626b3981f
commit 1ff30b9eaf
7 changed files with 248 additions and 58 deletions
+6 -6
View File
@@ -99,8 +99,8 @@ def exportCore(bugName):
if os.path.exists(join(BUGDIR, bugName,'diffs')):
shutil.rmtree(join(BUGDIR, bugName,'diffs'))
# os.makedirs(join(BUGDIR, bugName,'diffs'))
if os.path.exists(join(BUGDIR, bugName, 'src')):
shutil.rmtree(join(BUGDIR, bugName,'src'))
# if os.path.exists(join(BUGDIR, bugName, 'src')):
# shutil.rmtree(join(BUGDIR, bugName,'src'))
# os.makedirs(join(BUGDIR, bugName, 'src'))
#
cmd = 'docker create -ti --name dummy '+output.strip()+' bash'
@@ -114,10 +114,10 @@ def exportCore(bugName):
cmd = 'docker cp dummy:/experiment/src '+join(BUGDIR,bugName)
logging.info(cmd)
output, e = shellGitCheckout(cmd)
logging.info(output)
# cmd = 'docker cp dummy:/experiment/src '+join(BUGDIR,bugName)
# logging.info(cmd)
# output, e = shellGitCheckout(cmd)
# logging.info(output)
cmd = 'docker cp dummy:/experiment/diffs '+join(BUGDIR,bugName)
logging.info(cmd)
+13 -3
View File
@@ -33,7 +33,7 @@ if __name__ == '__main__':
# subject = 'ALL'
# rootType = 'if'
# job = 'patterns'
job = 'indexClusters'
print(job)
@@ -107,6 +107,12 @@ if __name__ == '__main__':
from sprinferIndex import removeDuplicates
removeDuplicates()
elif job =='mergeCoccis':
# from sprinferIndex import mergeCoccis
# mergeCoccis()
from sprinferIndex import removeDuplicates2
removeDuplicates2()
elif job == 'evalManyBugs':
# from patchManyBugs import patchCore
# patchCore()
@@ -176,8 +182,12 @@ if __name__ == '__main__':
from stats import defects4jStats
defects4jStats()
elif job == 'patterns':
from stats import exportAbstractPatterns
exportAbstractPatterns()
coccis = load_zipped_pickle(join(DATA_PATH, 'allCocciPatterns2.pickle'))
coccis['family'] = coccis.cid.apply(lambda x: x.split('.cocci')[0])
# from stats import exportAbstractPatterns
# exportAbstractPatterns()
elif job == 'travis':
if isfile(join(DATA_PATH,'repoList')):
+57 -27
View File
@@ -4,7 +4,7 @@ import csv
import os
from common.commons import *
DATA_PATH = os.environ["DATA_PATH"]
ROOT_DIR = os.environ["ROOT_DIR"]
def readTestSuite(testPath):
regex = r"([p|n0-9]+)\)"
with open(testPath,mode='r') as testFile:
@@ -127,7 +127,15 @@ def myvalidateCore(t):
def validateCore(t):
bugName, port = t
container = None
with bugzoo.server.ephemeral(port=port, verbose=False, timeout_connection=30000) as client:
cmd = 'bash {} {}'.format(join(ROOT_DIR, 'data', 'startBugzoo.sh'), port)
with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) as myProcess:
# o,e = shellGitCheckout(cmd)
url = "http://127.0.0.1:{}".format(port)
timeout_connection = 3000
client = Client(url, timeout_connection=timeout_connection)
# with bugzoo.server.ephemeral(port=port, verbose=False, timeout_connection=30000) as client:
try:
bug = client.bugs[bugName]
@@ -144,22 +152,22 @@ def validateCore(t):
preId, postId = bugName.split(':')[-1].split('-')[-2:]
failing = join(DATA_PATH, 'manybugs', bugName, 'failing.tests.txt')
passing = join(DATA_PATH, 'manybugs', bugName, 'passing.tests.txt')
if not (isfile(failing)):
return
cmd = 'docker cp ' +failing+ ' '+container.id +':/experiment/failing.tests.txt '
o,e = shellGitCheckout(cmd)
cmd = 'sudo chown $(whoami):$(whoami) "{}"'
cmd = cmd.format('failing.tests.txt')
o = client.containers.exec(container=container, command=cmd, context='/experiment/')
cmd = 'docker cp ' +passing+ ' '+container.id +':/experiment/passing.tests.txt '
o,e = shellGitCheckout(cmd)
cmd = 'sudo chown $(whoami):$(whoami) "{}"'
cmd = cmd.format('passing.tests.txt')
o = client.containers.exec(container=container, command=cmd, context='/experiment/')
# failing = join(DATA_PATH, 'manybugs', bugName, 'failing.tests.txt')
# passing = join(DATA_PATH, 'manybugs', bugName, 'passing.tests.txt')
#
# if not (isfile(failing)):
# return
#
# cmd = 'docker cp ' +failing+ ' '+container.id +':/experiment/failing.tests.txt '
# o,e = shellGitCheckout(cmd)
# cmd = 'sudo chown $(whoami):$(whoami) "{}"'
# cmd = cmd.format('failing.tests.txt')
# o = client.containers.exec(container=container, command=cmd, context='/experiment/')
# cmd = 'docker cp ' +passing+ ' '+container.id +':/experiment/passing.tests.txt '
# o,e = shellGitCheckout(cmd)
# cmd = 'sudo chown $(whoami):$(whoami) "{}"'
# cmd = cmd.format('passing.tests.txt')
# o = client.containers.exec(container=container, command=cmd, context='/experiment/')
originalBugs = get_filepaths(join(DATA_PATH, 'manybugs', bugName, 'diffs'), preId)
for ob in originalBugs:
@@ -180,7 +188,25 @@ def validateCore(t):
filepath = ob.split('diffs')[-1]
cmd = 'cp diffs' + filepath + ' src' + filepath.replace('-' + postId, '')
client.containers.exec(container=container, command=cmd, context='/experiment/')
client.containers.build(container)
times = 0
name = ob
patch_result = client.containers.build(container)
if patch_result.successful:
failure_cases, failure, total, test_outcomes = test_all(bug, container, client,validTests)
if failure == 0:
times += 1
output += ' fixed by {}'.format(name)
else:
output += ' {}'.format(failure_cases)
else:
output += ' {}'.format('build error')
output += ' times:{}'.format(times)
if times > 0:
output += ' success'
else:
output += ' failure'
# client.containers.build(container)
# diff_files = get_diff(bug.name, client, container)
# patch_result = patch_application(client, container, diff_files)
# if not patch_result:
@@ -191,13 +217,13 @@ def validateCore(t):
#
# print("Second_test:",end=' ')
# post_test_outcomes = {}
post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client,validTests)
output += ','.join(post_failure) + ' '
if len(post_failure) == 0:
# times += 1
fix = 'success'
# print("fix {} by {}".format(bugName, patch_name))
output += 'fix {} '.format(bugName)
# post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client,validTests)
# output += ','.join(post_failure) + ' '
# if len(post_failure) == 0:
# # times += 1
# fix = 'success'
# # print("fix {} by {}".format(bugName, patch_name))
# output += 'fix {} '.format(bugName)
# patch_names = os.listdir(join(DATA_PATH,'manybugs',bugName,'patched'))
@@ -245,6 +271,10 @@ def validateCore(t):
# ''
cmd = 'docker stop {}'.format(container.id)
out, e = shellGitCheckout(cmd)
try:
client.shutdown()
except Exception as e:
logging.error(e)
# docker stop $(docker ps -q)
@@ -276,7 +306,7 @@ def patch_validate():
if isinstance(v,logging.Logger):
v.setLevel(logging.ERROR)
buglist = listdir(join(DATA_PATH,'manybugs'))
buglist = listdir(join(DATA_PATH,'manybugs_sos'))
# for i in range(0,len(bugList)):
bugList = []
+90
View File
@@ -5,6 +5,7 @@ SPINFER_PATH = os.environ["spinfer"]
SPINFER_INDEX_PATH = os.environ["dataset"]
COCCI_PATH = join(os.environ["coccinelle"],'spatch')
DATASET = os.environ["dataset"]
ROOT_DIR = os.environ["ROOT_DIR"]
def indexCore():
@@ -157,6 +158,43 @@ def runSpinfer():
# output,e = shellGitCheckout(cmd)
# logging.info(output)
def mergeCoccis():
DATA = '/Users/anil.koyuncu/projects/fixminer/fixminer-data2'
folders = os.listdir(DATA)
coccis = [i for i in folders if 'cocci' in i]
allCoccis = [get_filepaths(join(DATA, i), '') for i in coccis]
for cocci in list(itertools.chain.from_iterable(allCoccis)):
_,srcFolder,cocciName = cocci.split(DATA)[-1].split('/')
with open(cocci, 'r') as iFile:
idx = iFile.readlines()
idx
values = np.array(idx)
points = np.where(values == '@@\n')
points = list(itertools.chain.from_iterable(points))
# if len(points) == 0:
# os.remove(join(SPINFER_INDEX_PATH, 'cocci', cocci))
# patches = list(zip(*([iter(points)] * 2)))
patches = list(pairwise(points[::2]))# every second element in list
if len(patches) > 0:
i = 0;
for t in patches :
t
with open(join(DATA, 'merged', cocciName+str(i)+'-'+srcFolder), 'w') as iFile:
iFile.writelines(idx[t[0]:t[1]])
i=i+1
t
with open(join(DATA, 'merged', cocciName + str(i)+'-'+srcFolder), 'w') as iFile:
iFile.writelines(idx[t[1]:])
else:
shutil.copy2(cocci,join(DATA, 'merged', cocciName + '-'+srcFolder))
def divideCoccis():
import datetime
shutil.copytree(join(SPINFER_INDEX_PATH,'cocci'),join(DATA_PATH,'cocci'+ datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')))
@@ -200,6 +238,58 @@ def getFreqPatterns():
re.search(r"// Recall:(.*), Precision:(.*), Matching recall:(.*)")
def removeDuplicates2():
commentPattern = r"(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)"
DATA = '/Users/anil.koyuncu/projects/fixminer/fixminer-data2'
coccis =os.listdir(join(DATA, 'merged'))
cocciPatterns = pd.DataFrame(columns=['cid', 'pattern','inferedFrom','recall','precision','matchingRecall'])
ind = 0
for cocci in coccis:
try:
if cocci == '.DS_Store':
continue
with open(join(DATA, 'merged', cocci), 'r') as iFile:
idx = iFile.read()
idx
inferedFrom = re.search(r"// Infered from:(.*)\n",idx).groups()
recall,precision, matchingRecall = re.search(r"// Recall:(.*), Precision:(.*), Matching recall:(.*)",idx).groups()
pattern = re.sub(commentPattern, '', idx, re.DOTALL)
cocciPatterns.loc[ind] = [cocci,pattern,inferedFrom,recall.strip(),precision.strip(),matchingRecall.strip()]
ind = ind +1
except Exception as e:
shutil.move(join(DATA,'merged',cocci),join(DATA,'mergedBroken',cocci))
continue
# logging.error(e)
cocciPatterns['iFiles'] = cocciPatterns.inferedFrom.apply(lambda x: getInferred(x[0]))
cocciPatterns['freq'] = cocciPatterns.iFiles.apply(lambda x: len(x))
cocciPatterns['project'] = cocciPatterns.iFiles.apply(lambda x: list(set([i.split('/{')[0].replace('(','') for i in x])))
cocciPatterns.sort_values(by='freq', inplace=True, ascending=False)
# save_zipped_pickle(cocciPatterns,join(DATA_PATH,'allCocciPatterns.pickle'))
save_zipped_pickle(cocciPatterns,join(DATA,'allCocciPatternsMerge.pickle'))
cocciPatterns['dup'] = cocciPatterns.duplicated(subset=['pattern'], keep=False)
duplicates = cocciPatterns[cocciPatterns.dup == True]
duplicateDF = duplicates.groupby(by=['pattern'], as_index=False).agg(lambda x: x.tolist())
pattern2keep= duplicateDF.cid.apply(lambda x: min(x, key=len)).values.tolist()
allDuplicates = list(itertools.chain.from_iterable(duplicateDF.cid.values.tolist()))
toRemove = list(set(allDuplicates).difference(set(pattern2keep)))
for tr in toRemove:
shutil.move(join(DATA,'merged',tr),join(DATA,'mergedDuplicate',tr))
# allPatterns = cocciPatterns.cid.values.tolist()
# uniquePatterns = cocciPatterns.drop_duplicates(subset=['pattern']).cid.values.tolist()
# toRemove = list(set(allPatterns).difference(uniquePatterns))
# print(toRemove)
# for p in toRemove:
# os.remove(join(SPINFER_INDEX_PATH, 'cocci', p))
# print(len(uniquePatterns))
def removeDuplicates():
commentPattern = r"(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)"
coccis =os.listdir(join(SPINFER_INDEX_PATH, 'cocci'))
+10 -4
View File
@@ -473,12 +473,18 @@ def exportAbstractPatterns():
if 'java' == PROJECT_TYPE:
isJava = True
for id, members in df[['cid','members']].values.tolist():
if (len(members) == 0):
continue
try:
dKey = '/'.join(id[0].split('-')[:-1]) + "/" + members[0]
lines = redis_db.hget("dump",dKey )
dKey = '/'.join(id[0].split('-')[:-1]) + "/" + members[0]
lines = redis_db.hget("dump",dKey )
cid = id[0].replace("-",'#')
abstractPattern(cid,lines.decode(),isJava,members)
cid = id[0].replace("-",'#')
abstractPattern(cid,lines.decode(),isJava,members)
except Exception as e:
logging.error(e)
def abstractPattern(cid,lines,isJava,cMembers):
+71 -17
View File
@@ -6,7 +6,34 @@ import signal
# from common.commons import shellGitCheckout
DATA_PATH = os.environ["DATA_PATH"]
ROOT_DIR = os.environ["ROOT_DIR"]
DATASET = os.environ["dataset"]
introClassFile = join(DATA_PATH,'introClassData.txt')
COCCI_PATH = join(os.environ["coccinelle"],'spatch')
def patchSourceFile(bugPath,spfile,bugName):
# print(bugPath)
srcName = bugPath.split('/')[-1].split(':')[1]
srcPath = join(bugPath,srcName + '.c')
patchName = srcName+'.c'
if(isfile(join(DATA_PATH,"introclass",bugName,'patched',patchName+spfile+'.c'))):
return join(DATA_PATH,"introclass",bugName,'patched',patchName+spfile+'.c')
if not (isfile(join(DATA_PATH,"introclass",bugName,'patches',patchName+spfile+'.txt'))):
cmd = COCCI_PATH + ' --sp-file ' + join(DATASET, 'cocci', spfile) + ' ' + srcPath + ' --patch -o' + join(
DATA_PATH, "introclass", bugName, 'patches', patchName) + ' > ' + join(DATA_PATH, "introclass", bugName,
'patches',
patchName + spfile + '.txt')
output, e = shellGitCheckout(cmd)
# logging.info(output)
patchSize = os.path.getsize(join(DATA_PATH,"introclass",bugName,'patches',patchName+spfile+'.txt'))
if patchSize == 0 :
# os.remove(join(DATA_PATH,"introclass",bugName,'patches',patchName+spfile+'.txt'))
return None
else:
cmd = 'patch -d '+join(DATA_PATH, "introclass", bugName)+' -i '+join(DATA_PATH,"introclass",bugName,'patches',patchName+spfile+'.txt')+' -o '+join(DATA_PATH,"introclass",bugName,'patched',patchName+spfile+'.c')
o,e = shellGitCheckout(cmd)
return join(DATA_PATH, "introclass", bugName, 'patched', patchName + spfile + '.c')
def testCore(t):
bugName, port = t
@@ -14,7 +41,7 @@ def testCore(t):
# with bugzoo.server.ephemeral(port=port, verbose=False,bugzooPath="/Users/anil.koyuncu/anaconda3/envs/python36/bin/bugzood", timeout_connection=3000) as client:
cmd = 'bash {} {}'.format(join(ROOT_DIR,'data','startBugzoo.sh'),port)
with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) as process:
with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) as myProcess:
# o,e = shellGitCheckout(cmd)
url = "http://127.0.0.1:{}".format(port)
@@ -56,27 +83,39 @@ def testCore(t):
return ''
output += '@fail:' + str(pre_failure) + '@total:' + str(total) + ', '
# print("patching...")
path = join(DATA_PATH,'introclass',bugName)
patch_path = join(path ,'patched')
# avaliable_patch = os.path.abspath('data') + '/introclass2/' + bugName + '/' + 'patches'
patch_names = os.listdir(patch_path)
spfiles = listdir(join(DATASET, 'cocci'))
# print("patching... " + bugName)
for idx,spfile in enumerate(spfiles):
if spfile == '.DS_Store':
continue
times = 0
for patch_name in patch_names:
path = join(DATA_PATH,'introclass',bugName)
patch = patchSourceFile(path,spfile,bugName)
if patch is None:
continue
# print(patch)
# print(str(idx) + '..'+str(len(spfiles)))
# patch_path = join(path ,'patched')
# # avaliable_patch = os.path.abspath('data') + '/introclass2/' + bugName + '/' + 'patches'
# patch_names = os.listdir(patch_path)
#
times = 0
# for patch_name in patch_names:
# if patch_name not in os.listdir(avaliable_patch):
# continue
patch = join(patch_path,patch_name)
# patch = join(patch_path,patch_name)
patch_result = patched_application(path, bug.name, patch, client, container)
if patch_result == -1 or patch_result.code != 0:
# print("@{}@".format('False'), end='')
# print("{}".format('F'), end=' ')
output += '@False@F '
# output += '@False@F '
output += '@False:' + str(idx) + ':' + patch.split('/')[-1] + '@'
continue
# print("@{}@".format('True'), end='')
output += '@True@'
# output += '@True@'
output += '@True:' + str(idx) + ':' + patch.split('/')[-1] + '@'
# print("Second_test:", end=' ')
post_test_outcomes = {}
post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client)
@@ -86,7 +125,8 @@ def testCore(t):
times += 1
fix = 'success'
# print("fix {} by {}".format(bugName, patch_name))
output += 'fix {} by {} '.format(bugName, patch_name)
output += 'fix {} by {} '.format(bugName, patch)
break
# print("@fail:{}@total:{}".format(post_failure, total),end=' ')
# print("@post_failure_cases:{}".format(post_failure_cases))
@@ -103,8 +143,12 @@ def testCore(t):
# ''
cmd = 'docker stop {}'.format(container.id)
out, e = shellGitCheckout(cmd)
client.shutdown()
os.killpg(process.pid, signal.SIGTERM)
try:
client.shutdown()
except Exception as e:
logging.error(e)
# print(myProcess.pid)
# os.killpg(myProcess.pid, signal.SIGTERM)
# docker stop $(docker ps -q)
@@ -124,6 +168,10 @@ def patch_validate():
# bug = client.bugs['introclass:checksum:08c7ea:006']
bugList = []
nonFail = ['introclass:syllables:93f87b:005','introclass:smallest:f94e26:004','introclass:smallest:d25c71:001','introclass:median:d25c71:002','introclass:smallest:d25c71:000'
,'introclass:median:d6364e:007','introclass:median:489253:007','introclass:syllables:d12048:004','introclass:smallest:d9e7ea:002','introclass:syllables:035fe9:000'
,'introclass:syllables:c9d718:002','introclass:syllables:ea67b8:007','introclass:median:48b829:000','introclass:syllables:d9e7ea:001']
# cmd = 'bash ' + join(DATA_PATH,'startBugzoo.sh')
# cmd = "/Users/anil.koyuncu/anaconda3/envs/python36/bin/bugzood --debug -p " + str(port)
# output, errors = shellGitCheckout(cmd)
@@ -131,6 +179,11 @@ def patch_validate():
port = 6000
bugs2test= listdir(join(DATA_PATH, "introclass"))
for b in bugs2test:
if b == '.DS_Store':
continue
if b in nonFail:
continue
t = b, port
bugList.append(t)
if port == 6300:
@@ -149,8 +202,9 @@ def patch_validate():
# t = 'introclass:syllables:93f87b:005',6000
# t = 'introclass:syllables:99cbb4:000',6000
# testCore(t)
results = parallelRunMerge(testCore, bugList,max_workers=10)
# print('\n'.join(results))
# results = parallelRunMerge(testCore, bugList,max_workers=10)
results = parallelRunMerge(testCore, bugList)
print('\n'.join(results))
with open(join(DATA_PATH, 'introTestResults'), 'w',
encoding='utf-8') as writeFile:
# if levelPatch == 0:
+1 -1
View File
@@ -75,7 +75,7 @@ def patch_validate(t):
finally:
cmd = 'docker stop {}'.format(container.id)
out, e = shellGitCheckout(cmd)
client.shutdown()
# client.shutdown()
os.killpg(process.pid, signal.SIGTERM)
from bugzoo import Patch, Client