merge python scripts

.idea/workspace.xml

@@ -2,7 +2,7 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="75ef5724-93bc-4ed4-82b2-ff12411188dc" name="Default Changelist" comment="">
-      <change beforePath="$PROJECT_DIR$/.travis.yml" beforeDir="false" afterPath="$PROJECT_DIR$/.travis.yml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
     </list>
     <ignored path="$PROJECT_DIR$/classes/" />
     <ignored path="$PROJECT_DIR$/target/" />
@@ -48,7 +48,7 @@
           </provider>
         </entry>
       </file>
-      <file pinned="false" current-in-tab="true">
+      <file pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/.travis.yml">
           <provider selected="true" editor-type-id="text-editor">
             <state relative-caret-position="45">
@@ -57,6 +57,11 @@
           </provider>
         </entry>
       </file>
+      <file pinned="false" current-in-tab="true">
+        <entry file="file://$PROJECT_DIR$/README.md">
+          <provider selected="true" editor-type-id="text-editor" />
+        </entry>
+      </file>
     </leaf>
   </component>
   <component name="Git.Settings">
@@ -175,17 +180,16 @@
   </component>
   <component name="ToolWindowManager">
     <frame x="0" y="0" width="1680" height="1050" extended-state="0" />
-    <editor active="true" />
     <layout>
       <window_info id="Image Layers" />
       <window_info id="Designer" />
       <window_info id="UI Designer" />
       <window_info id="Capture Tool" />
       <window_info id="Favorites" side_tool="true" />
-      <window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.4035409" />
+      <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.4035409" />
       <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
       <window_info anchor="bottom" id="Version Control" />
-      <window_info active="true" anchor="bottom" id="Terminal" visible="true" weight="0.32959184" />
+      <window_info anchor="bottom" id="Terminal" visible="true" weight="0.32959184" />
       <window_info anchor="bottom" id="Event Log" side_tool="true" />
       <window_info anchor="bottom" id="Message" order="0" />
       <window_info anchor="bottom" id="Find" order="1" />
@@ -242,6 +246,9 @@
         </state>
       </provider>
     </entry>
+    <entry file="file://$PROJECT_DIR$/README.md">
+      <provider selected="true" editor-type-id="text-editor" />
+    </entry>
   </component>
   <component name="masterDetails">
     <states>

README.md

@@ -4,4 +4,235 @@
 
 Reference: [FixMiner: Mining Relevant Fix Patterns for Automated Program Repair](http://arxiv.org/pdf/1810.01791) (Empirical Software Engineering, [doi:10.1007/s10664-019-09780-z](https://doi.org/10.1007/s10664-019-09780-z))
 
+# FixMiner
+
+* [I. Introduction of FixMiner](#user-content-i-introduction)
+* [II. Environment setup](#user-content-ii-environment)
+* [III. Replication Data](#user-content-iii-data)
+* [IV. Step-by-Step execution](#user-content-iv-how-to-run)
+<!--
+* [V. Evaluation Result](#user-content-v-evaluation-result)
+* [VI. Generated Patches](#user-content-vi-generated-patches)
+* [VII. Structure of the project](#user-content-vii-structure-of-the-project)
+-->
+
+## I. Introduction
+
+Fixminer is a systematic and automated approach to mine relevant and actionable fix patterns for automated program repair.
+![The workflow of this technique.\label{workflow}](worflow.png)
+
+## II. Environment setup
+
+* OS: macOS Mojave (10.14.3)
+* JDK8: Oracle jdk1.8 (**important!**)
+* Download and configure Anaconda
+* Create an python environment using the [environment file](environment.yml)
+  ```powershell
+  conda env create -f environment.yml
+  ```
+* After creating the environment, activate it. It is containing necessary dependencies for redis, and python.
+  ```powershell
+  source activate redisEnv
+  ```
+<!---
+[fixminer.sh](python/fixminer.sh)
+
+Unzip it,to the datasetPath path indicated in app.properties.
+
+    7z x allDataset.7z
+    
+In order to launch FixMiner, execute [fixminer.sh](python/fixminer.sh)
+
+    bash fixminer.sh /Users/..../enhancedASTDiff/python/ stats
+--->
+    
+## III. Replication Data
+Replication Data:
+    
+   [singleBR.pickle](python/data/singleBR.pickle)
+    
+    This pickle contains the list bug reports (i.e. bid) with the their corresponding fixes (i.e. commit) for each project in the dataset (i.e. project). 
+    
+   [bugReports.7z.00X](python/data/bugReports.7z.001)
+   
+    This is the dump of the bug reports archive extracted from each commit. These bug reports are not necessarily considered as BUG,CLOSED; this archive is the contins initial bug reports before identifying the fixes. 
+    
+   [gumInput.7z.001](python/data/gumInput.7z.001)
+   
+    This archive contains all the patches in our dataset, formatted in a way that can be processed by GumTree (i.e DiffEntries, prevFiles, revFiles)
+    
+   [ALLbugReportsComplete.pickle](python/data/ALLbugReportsComplete.pickle)
+   
+    The pickle object that represents the bug reports under the following columns 'bugReport', 'summary', 'description', 'created', 'updated', 'resolved', 'reporterDN', 'reporterEmail','hasAttachment', 'attachmentTime', 'hasPR', 'commentsCount'
+
+#### Data Viewer
+
+The data provided with replication package is listed in directory [python/data](python/data)
+The data is stored in different formats. (e.g. pickle, db, csv, etc..)
+
+The see content of the .pickle file the following script could be used.
+
+  ```python
+   import pickle as p
+   import gzip
+   def load_zipped_pickle(filename):
+      with gzip.open(filename, 'rb') as f:
+          loaded_object = p.load(f)
+          return loaded_object
+  ```
+Usage
+
+  ```python
+  result = load_zipped_pickle('code/LANGbugReportsComplete.pickle')
+  # Result is pandas object which can be exported to several formats
+  # Details on how to export is listed in offical library documentation
+  # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html
+
+  ```
+
+
+## IV. Step-by-Step execution
+
+#### Before running
+
+* Update [config file](python/config.yml) with corresponding user paths.
+
+* Active the conda environment from shell
+  ```powershell
+  source activate redisEnv
+  ```
+
+In order to launch FixMiner, execute [fixminer.sh](python/fixminer.sh)
+
+    bash fixminer.sh [PATH_TO_PYTHON_FOLDER] [OPTIONS]
+     e.g. bash fixminer.sh Users/fixminer-core/python/ stats
+    
+#### Running Options 
+
+*FixMiner* needs to specify an option to run.
+
+    1. 'dataset': Create a mining dataset from the project speficied in  [subjects.csv](python/data/subjects.csv)
+    Eventually dataset option is the execution of the following steps, which are merged under 'dataset' option 
+    for this demo. Eventually single options can be activated by commenting out the corresponding option in [main.py](python/main.py)
+
+        `clone` : Clone target project repository.
+    
+        `collect` : Collect all commit from repository.
+    
+        `fix` : Collect commits linked to a bug report.
+    
+        `bugPoints` : Identify the snapshot of the repository before the bug fixing commit introducted.
+    
+        `brDownload` : Download bug reports recovered from commit log
+    
+        `brParser` : Parse bug reports to select the bug report where type labelled as BUG and status as RESOLVED or CLOSED
+        
+    2. 'richEditScript': Rich edit script computation step.    
+    
+    3. 'shapeSI': Search index creation for shapes. The output of this step is written to [pairs](python/data/pairs)
+    
+    4. 'compareShapes' : ShapeTree comparison
+    
+    5. 'cluster': Forms clusters of identical shapetree. The output of this step is written to [shapes](python/data/shapes)
+    
+    6. 'actionSI': Search index creation for actions. The output of this step is written to [pairs](python/data/pairsAction)
+    
+    7. 'compareActions' : ActionTree comparison
+    
+    8. 'clusterActions': Forms clusters of identical ActionTree. The output of this step is written to [shapes](python/data/actions)
+    
+    9. 'tokenSI': Search index creation for shapes. The output of this step is written to [pairs](python/data/pairsToken)
+    
+    10. 'compareTokens' : TokenTree comparison
+    
+    11. 'clusterTokens': Forms clusters of identical TokenTree. The output of this step is written to [shapes](python/data/tokens)
+    
+    12. 'stats' : Calculate some statistics about patterns under python/data/statsactions.csv,statsshapes.csv,statstokens.csv, and export FixPatterns of APR integration [fixpatterns](actionPattern2verify.csv)
+
+
+<!--
+App.properties:
+
+
+FixMiner consists of several jobs that needs to run in order to extract fix pattern from the dataset.
+It is necessary to run the FixMiner, following the order.
+  1.ENHANCED AST DIFF calcuation
+
+    By setting the jobType = ENHANCEDASTDIFF. This will create the ENHANCEDASTDIFF for the dataset regardless of the actionType.
+
+  2.CACHE the enhanced AST Diff into memory cache
+
+    By setting the jobType = CACHE
+    
+  3.SI search index construction.
+  
+    By setting the jobType = SI
+    
+  4.SIMI in order to compare the similarity between the trees.
+  
+    By setting the jobType = SIMI
+
+  5.LEVEL1 mining
+
+    By setting the jobType = LEVEL1
+
+  6.LEVEL2 mining
+
+    By setting the jobType = LEVEL2
+
+  7.LEVEL3 mining
+
+    By setting the jobType = LEVEL3
+    
+    
+ A mining is iteration is executed for the actionType. In order to execute for all the actionTypes, the iteration should be repeated from 2-7 by changing the actionType.
+    
+  There are some additional parameters in the app.config. 
+  
+  actionType
+    
+    The admitted values are UPD,INS,DEL,MOV,MIX, which represents the ENHANCEDASTDIFF actions.
+    UPD/INS/DEL/MOV considers tree where a single action operation is done in the action set
+    MIX considers any action.
+  
+  parallelism
+    
+    The engine to use for parallelism. It is either FORKJOIN or AKKA. 
+    FORKJOIN is recommended is the FixMiner is running on a single machine. 
+    AKKA is suggested for distributed machines.
+    
+  numOfWorkers
+  
+    The number of workers that will be generated when AKKA is selected as the parallelism engine.
+    
+  cursor
+  
+    The maximum number of pairs in during the search index SI creation.
+    
+   eDiffTimeout
+   
+    The timeout value in seconds for the Enhanced Diff computation (ENHANCEDASTDIFF). 
+    In case ENHANCEDASTDIFF step logs timeouts, this value can be increase. 
+    
+    
+   The following parameters should be used when dealing with extremely large dataset. Otherwise, default values are suggested.
+    
+   isBigPair
+    
+    This flag when set to true, splits the pairs that into chunks as ..0.txt,1.txt etc. 
+    
+   chunk
+   
+    The extension of the pairs files. When isBigPair is set to false(which is default), it needs to be set as .csv 
+    When isBigPair mode is activated then the SIMI step executed for each chunk by stepping the chunk as 0.txt, 1,txt) 
+
+    
+## V. Evaluation Result
+## VI. Generated Patches
+## VII. Structure of the project
+--> 
+    
+
+
+
 

environment.yml

@@ -0,0 +1,13 @@
+name: fixminerEnv
+channels:
+  - anaconda
+  - defaults
+dependencies:
+  - numpy
+  - tqdm
+  - pandas
+  - pyyaml
+  - redis-py
+  - scikit-learn
+  - networkx=2.1
+

python/MF0012.local.config.yml

@@ -0,0 +1,16 @@
+java:
+    8home:  /Library/Java/JavaVirtualMachines/jdk1.8.0_181.jdk/Contents/Home
+spinfer:
+    home:   /Users/anil.koyuncu/projects/spinfer/spinfer.native
+dataset:
+    home:   /Users/anil.koyuncu/projects/test/fixminer-data/patches/
+    repo:   /Users/anil.koyuncu/projects/test/fixminer-data/datasets
+#    home:   /data/fixminer-core/python/data/gumInputLinux/
+
+coccinelle:
+    home:   /Users/anil.koyuncu/projects/spinfer/statics
+
+fixminer:
+    datapath: /Users/anil.koyuncu/projects/test/fixminer-data/
+    projectType : c
+#    datapath: /data/fixminer-core/python/data/

python/abstractPatch.py

@@ -0,0 +1,306 @@
+
+import redis
+
+from common.commons import *
+
+DATA_PATH = os.environ["DATA_PATH"]
+DATASET = os.environ["dataset"]
+jdk8 = os.environ["JDK8"]
+# def localPairCore(aTuple):
+#     redis_db = redis.StrictRedis(host="localhost", port=6380, db=1)
+#     idx, key = aTuple
+#
+#
+#
+#     val = redis_db.get(key)
+#
+#     res = val.decode().split(',')
+#     res.insert(0, key.decode().split('_')[1:])
+#     res.insert(0, key.decode())
+#     # matches.loc[idx] = res
+#     return res
+ast = ["AnonymousClassDeclaration", "ArrayAccess", "ArrayCreation", "ArrayInitializer", "ArrayType", "AssertStatement",
+       "Assignment", "Block", "BooleanLiteral", "BreakStatement", "CastExpression", "CatchClause", "CharacterLiteral",
+       "ClassInstanceCreation", "CompilationUnit", "ConditionalExpression", "ConstructorInvocation",
+       "ContinueStatement", "DoStatement", "EmptyStatement", "ExpressionStatement", "FieldAccess", "FieldDeclaration",
+       "ForStatement", "IfStatement", "ImportDeclaration", "InfixExpression", "Initializer", "Javadoc",
+       "LabeledStatement", "MethodDeclaration", "MethodInvocation", "NullLiteral", "NumberLiteral",
+       "PackageDeclaration", "ParenthesizedExpression", "PostfixExpression", "PrefixExpression", "PrimitiveType",
+       "QualifiedName", "ReturnStatement", "SimpleName", "SimpleType", "SingleVariableDeclaration", "StringLiteral",
+       "SuperConstructorInvocation", "SuperFieldAccess", "SuperMethodInvocation", "SwitchCase", "SwitchStatement",
+       "SynchronizedStatement", "ThisExpression", "ThrowStatement", "TryStatement", "TypeDeclaration",
+       "TypeDeclarationStatement", "TypeLiteral", "VariableDeclarationExpression", "VariableDeclarationFragment",
+       "VariableDeclarationStatement", "WhileStatement", "InstanceofExpression", "LineComment", "BlockComment",
+       "TagElement", "TextElement", "MemberRef", "MethodRef", "MethodRefParameter", "EnhancedForStatement",
+       "EnumDeclaration", "EnumConstantDeclaration", "TypeParameter", "ParameterizedType", "QualifiedType",
+       "WildcardType", "NormalAnnotation", "MarkerAnnotation", "SingleMemberAnnotation", "MemberValuePair",
+       "AnnotationTypeDeclaration", "AnnotationTypeMemberDeclaration", "Modifier", "UnionType", "Dimension",
+       "LambdaExpression", "IntersectionType", "NameQualifiedType", "CreationReference", "ExpressionMethodReference",
+       "SuperMethodReference", "TypeMethodReference", "MethodName", "Operator", "New", "Instanceof"]
+
+ast = ["unit","comment","name","type","condition","block","index","decltype","typename","atomic","assert","generic_selection","selector","association_list",
+       "association","expr_stmt","expr","decl_stmt","decl","range","break","continue","goto","label","typedef","asm","enum","ternary","elseif","while","lock",
+       "fixed","checked","unchecked","unsafe","do","switch","case","default","for","foreach","group","init","incr","function","function_decl","lambda","specifier",
+       "return","call","sizeof","parameter_list","param","krparameter_list","krparam","argument_list","argument","capture","struct","struct_decl","union","union_decl",
+       "class","class_decl","public","private","protected","namespace","using","try","catch","finally","throw","throws","noexcept","template","directive","file","number",
+       "include","define","undef","line","if","ifdef","ifndef","else","elif","endif","then","pragma","error","macro","value","import","constructor_decl","empty_stmt","escape","annotation","alignof","forever","extern"]
+
+movPattern = 'MOV (' + '|'.join(ast) + ')@@(.*)@TO@ (' + '|'.join(ast) + ')@@(.*)@AT@'
+delPattern = 'DEL (' + '|'.join(ast) + ')@@(.*)@AT@'
+insPattern = 'INS (' + '|'.join(ast) + ')@@(.*)@TO@ (' + '|'.join(ast) + ')@@(.*)@AT@'
+updPattern = 'UPD (' + '|'.join(ast) + ')@@(.*)@TO@(.*)@AT@'
+
+def loadPairMulti(root,clusterPath,level):
+
+    # root = 'BreakStatement'
+    logging.info(root)
+    port = 6399
+    # if isfile(clusterPath +"/"+root+".pickle"):
+    #     return load_zipped_pickle(clusterPath +"/"+root+".pickle")
+    # else:
+        # redis_db = redis.StrictRedis(host="localhost", port=port, db=1)  #L1
+    redis_db = redis.StrictRedis(host="localhost", port=port, db=2)
+    keys = redis_db.scan(0, match=root+'-*', count='100000000')
+    # keys = redis_db.hkeys("dump")
+
+    # tuples = []
+    # for idx,key in enumerate(keys[1]):
+    #     t = idx,key
+    #     tuples.append(t)
+
+    # coreNumber = 1600
+    # print('Core number %s' % coreNumber)
+    matches = pd.DataFrame(keys[1],columns=['pairs_key'])
+    matches['pairs_key']=matches['pairs_key'].apply(lambda x:x.decode())
+    # matches['pairs']=matches['pairs_key'].apply(lambda x:x.split('_')[1:])
+    matches['pairs']=matches['pairs_key'].apply(lambda x:x.split(root)[1].split('/')[1:])
+    matches['tuples'] = matches.pairs.apply(lambda x: tuple(x))
+    # matches['path1']=matches['pairs_key'].apply(lambda x:x.split('_')[1])
+    matches['path1']=matches['pairs'].apply(lambda x:x[0])
+    # matches['path2']=matches['pairs_key'].apply(lambda x:x.split('_')[2])
+    matches['path2']=matches['pairs'].apply(lambda x:x[1])
+    # matches['sizes']=matches['pairs_key'].apply(lambda x:x.split('_')[0].split('-')[1])
+    matches['sizes']=matches['pairs_key'].apply(lambda x:x.split(root)[1].split('/')[0].split('-')[1])
+    if level == 'actions':
+        matches['actions']=matches['pairs_key'].apply(lambda x:x.split('/')[0].split('-')[2])
+    if level == 'tokens':
+        matches['actions'] = matches['pairs_key'].apply(lambda x: x.split('/')[0].split('-')[2])
+        matches['tokens']=matches['pairs_key'].apply(lambda x:x.split('/')[0].split('-')[3])
+
+
+    # save_zipped_pickle(matches,clusterPath +"/"+root+".pickle")
+    return matches
+
+
+
+def getMapping(pathMapping,x):
+    pair1,pair2 = x['pairs']
+    p1 = x['path1']
+    p2 = x['path2']
+    pathMapping[pair1] = p1
+    pathMapping[pair2] = p2
+
+
+
+
+
+def cluster(clusterPath,pairsPath, level,rootType):
+
+        try:
+            # logging.info('Parameters: \ninputPath %s \nclusterPath %s \nport %s \nmatchesName %s \nthreshold %s \n%indexFile',inputPath,clusterPath,str(port),matchesName,str(threshold),indexFile)
+            os.makedirs(clusterPath, exist_ok=True)
+            roots = listdir(pairsPath)
+            roots = [i for i in roots if not i.startswith('.')]
+            # roots = [rootType]
+            # parallelRun(loadPairMulti,roots,clusterPath)
+            for root in roots:
+                matches = loadPairMulti(root,clusterPath,level)
+                sizes = matches['sizes'].unique().tolist()
+                for s in sizes:
+                    match = matches[matches['sizes'] == s]
+
+                    if level == 'actions':
+                        actions = match['actions'].unique().tolist()
+                        for action in actions:
+                            match = match[match['actions'] == action]
+                            clusterCore(clusterPath,  level, match, pairsPath, root, s,action)
+                    elif level == 'tokens':
+                        actions = match['actions'].unique().tolist()
+                        for action in actions:
+                            match = match[match['actions'] == action]
+                            tokens = match['tokens'].unique().tolist()
+                            for token in tokens:
+                                match = match[match['tokens']==token]
+                                clusterCore(clusterPath, level, match, pairsPath, root, s, action,token)
+                    else:
+                        clusterCore(clusterPath,  level, match, pairsPath, root, s,'')
+
+            # redis_db = redis.StrictRedis(host="localhost", port=6399, db=2)
+            # redis_db.flushdb()
+            # redis_db = redis.StrictRedis(host="localhost", port=6399, db=1)
+            # redis_db.flushdb()
+            # redis_db = redis.StrictRedis(host="localhost", port=6399, db=0)
+            # redis_db.delete("compare")
+
+
+
+        except Exception as ex:
+            logging.error(ex)
+
+
+def clusterCore(clusterPath, level, match, pairsPath, root, s,action ,token=''):
+    col_combi = match.tuples.values.tolist()
+    import networkx
+    g = networkx.Graph(col_combi)
+    cluster = []
+    for subgraph in networkx.connected_component_subgraphs(g):
+        logging.info('Cluster size %d',len(subgraph.nodes()))
+        cluster.append(subgraph.nodes())
+    cluster
+    pathMapping = dict()
+    if level == 'actions':
+        indexFile = join(pairsPath, root, s,action+'.index')
+    elif level == 'shapes':
+        indexFile = join(pairsPath, root, s + '.index')
+    else:
+        indexFile =join(pairsPath, root, s,action,token+'.index')
+    df = pd.read_csv(indexFile, header=None, usecols=[0, 1], index_col=[0])
+    pathMapping = df.to_dict()
+
+    workList = []
+    for idx, clus in enumerate(cluster):
+        logging.info('exporting cluster %s %s %s %d', root,s,action,idx)
+        for f in clus:
+            dumpFile = pathMapping[1][int(f)]
+
+            t = dumpFile,root,level,clusterPath,s,action,token,idx
+            workList.append(t)
+
+    parallelRun(dumpFilesCore,workList)
+    # for wl in workList:
+    #     dumpFilesCore(wl)
+    # dumpFilesCore(('hive_d65d5c_96c1dc_ql#src#gen#protobuf#gen-java#org#apache#hadoop#hive#ql#io#orc#OrcProto.txt_31', 'ReturnStatement', 'tokens', '/Users/anil.koyuncu/projects/fixminer-all/enhancedASTDiff/python/data/tokens', '3', '1', '0', 10))
+
+
+def dumpFilesCore(t):
+
+    try:
+        dumpFile, root, level, clusterPath, s, action, token, idx = t
+        split = dumpFile.split('_')
+        project = split[0]
+        filename = "_".join(split[1:-1])
+        filePath = join(DATASET, project, 'DiffEntries', filename)
+
+        key = root + '/'+s+'/'+dumpFile
+        jdk8 = os.environ["JDK8"]
+        # cmd = "JAVA_HOME='"+jdk8+"' java -jar "+ join(DATA_PATH,'FixPatternMiner-1.0.1.jar') + " " + join(DATA_PATH,'app.properties')+" PATTERN " +key
+
+        clusterSavePath = join(clusterPath, root, s, str(idx))
+        os.makedirs(clusterSavePath, exist_ok=True)
+        shutil.copy(filePath,join(clusterSavePath,dumpFile))
+        # with open(join(clusterSavePath, dumpFile), 'w', encoding='utf-8') as writeFile:
+        #     writeFile.write(lines)
+
+
+
+        clusterSavePath = ''
+        # if level == 'shapes':
+        #     clusterSavePath = join(clusterPath, root,s, str(idx))
+        #
+        #     o, e = shellGitCheckout(cmd)
+        #     lines = o
+        # elif level == 'actions':
+        #     clusterSavePath = join(clusterPath, root, s,action, str(idx))
+        #
+        #     o, e = shellGitCheckout(cmd)
+        #     lines = o
+        # else:
+        #     clusterSavePath = join(clusterPath, root, s,action,token, str(idx))
+        #     o, e = shellGitCheckout(cmd)
+        #     lines = o
+        #     # with open(filePath, 'r', encoding='utf-8') as fi:
+        #     #     lines = fi.read()
+        #
+        # if level =='shapes' or level=='actions':
+        #
+        #     lines = re.split("@LENGTH@ \d+", lines)
+        #     tokens = []
+        #     for line in lines:
+        #         # levelPatch  = len(re.findall('\w*---', line))
+        #         match = re.search(r"^\w*---+", line,re.M)
+        #
+        #
+        #         if match is not None:
+        #             not_matched, matched = line[:match.start()], match.group()
+        #             levelPatch  = int(len(matched) / 3)
+        #         else:
+        #             levelPatch = 0
+        #         line = line.strip().strip('-')
+        #         type = ''
+        #         if line is '':
+        #             continue
+        #         t = []
+        #         searchPattern = ''
+        #         if line.startswith('INS'):
+        #             if level =='actions':
+        #                 t= [1,3]
+        #             else:
+        #                 t = [1]
+        #             searchPattern = insPattern
+        #             type =' INS '
+        #         elif line.startswith('UPD'):
+        #             t = [1]
+        #             searchPattern = updPattern
+        #             type = ' UPD '
+        #         elif line.startswith('DEL'):
+        #             t = [1]
+        #             searchPattern = delPattern
+        #             type = ' DEL '
+        #         elif line.startswith('MOV'):
+        #             if level == 'actions':
+        #                 t = [1, 3]
+        #             else:
+        #                 t = [1]
+        #             searchPattern = movPattern
+        #             type = ' MOV '
+        #     # from common.preprocessing import preprocessingForSimi
+        #         m = re.search(searchPattern, line, re.DOTALL)
+        #         if t is None:
+        #             print()
+        #         if m:
+        #             for k in t:
+        #                 prefix = '---' * levelPatch
+        #                 if prefix != '':
+        #                     prefix = '\n'+prefix
+        #                 token = m.group(k)
+        #                 if level =='actions':
+        #                     if k ==3:
+        #                         prefix = 'TO '
+        #                     else:
+        #                         prefix = prefix + type
+        #
+        #                 tokens.append(prefix+token)
+        #
+        #     os.makedirs(clusterSavePath, exist_ok=True)
+        #     with open(join(clusterSavePath, dumpFile), 'w', encoding='utf-8') as writeFile:
+        #         # if levelPatch == 0:
+        #         writeFile.write(' '.join(tokens))
+        #         # else:
+        #         #     writeFile.write('\n'.join(tokens))
+        # else:
+        #     os.makedirs(clusterSavePath, exist_ok=True)
+        #     with open(join(clusterSavePath, dumpFile), 'w', encoding='utf-8') as writeFile:
+        #         writeFile.write(lines)
+
+    except Exception as ex:
+        logging.error(t)
+        logging.error(ex)
+        raise ex
+
+
+
+
+
+
+
+

python/actionsPattern2verify.csv

@@ -0,0 +1,222 @@
+ExpressionStatement-3-0-0," UPD ExpressionStatement 
+--- UPD MethodInvocation 
+------ UPD SimpleName"
+ExpressionStatement-4-3-0," UPD ExpressionStatement 
+--- UPD MethodInvocation 
+------ UPD SimpleName 
+--------- UPD StringLiteral"
+VariableDeclarationStatement-3-0-0," UPD VariableDeclarationStatement 
+--- UPD VariableDeclarationFragment 
+------ UPD SimpleName"
+FieldDeclaration-2-0-0," UPD FieldDeclaration 
+--- INS Modifier TO FieldDeclaration"
+FieldDeclaration-6-0-0," INS FieldDeclaration TO TypeDeclaration 
+--- INS Modifier TO FieldDeclaration 
+--- INS Modifier TO FieldDeclaration 
+--- INS SimpleType TO FieldDeclaration 
+--- INS VariableDeclarationFragment TO FieldDeclaration 
+------ INS SimpleName TO VariableDeclarationFragment"
+FieldDeclaration-8-0-0," INS FieldDeclaration TO TypeDeclaration 
+--- INS Modifier TO FieldDeclaration 
+--- INS Modifier TO FieldDeclaration 
+--- INS Modifier TO FieldDeclaration 
+--- INS SimpleType TO FieldDeclaration 
+--- INS VariableDeclarationFragment TO FieldDeclaration 
+------ INS SimpleName TO VariableDeclarationFragment 
+------ INS StringLiteral TO VariableDeclarationFragment"
+ExpressionStatement-2-0-0," DEL ExpressionStatement 
+--- DEL MethodInvocation"
+ExpressionStatement-2-0-1," INS ExpressionStatement TO MethodDeclaration 
+--- INS MethodInvocation TO ExpressionStatement"
+VariableDeclarationStatement-5-2-0," INS VariableDeclarationStatement TO MethodDeclaration 
+--- INS SimpleType TO VariableDeclarationStatement 
+--- INS VariableDeclarationFragment TO VariableDeclarationStatement 
+------ INS SimpleName TO VariableDeclarationFragment 
+------ INS NullLiteral TO VariableDeclarationFragment"
+VariableDeclarationStatement-2-0-0," UPD VariableDeclarationStatement 
+--- INS Modifier TO VariableDeclarationStatement"
+MethodDeclaration-2-0-0," UPD MethodDeclaration 
+--- INS Modifier TO MethodDeclaration"
+VariableDeclarationStatement-7-0-0," INS VariableDeclarationStatement TO MethodDeclaration 
+--- INS SimpleType TO VariableDeclarationStatement 
+--- INS VariableDeclarationFragment TO VariableDeclarationStatement 
+------ INS SimpleName TO VariableDeclarationFragment 
+------ INS MethodInvocation TO VariableDeclarationFragment 
+--------- INS SimpleName TO MethodInvocation 
+--------- INS SimpleName TO MethodInvocation"
+FieldDeclaration-3-0-0," UPD FieldDeclaration 
+--- UPD VariableDeclarationFragment 
+------ UPD SimpleName"
+ExpressionStatement-5-6-0," INS ExpressionStatement TO IfStatement 
+--- INS Assignment TO ExpressionStatement 
+------ INS SimpleName TO Assignment 
+------ INS Operator TO Assignment 
+------ INS BooleanLiteral TO Assignment"
+VariableDeclarationStatement-4-0-0," DEL VariableDeclarationStatement 
+--- DEL SimpleType 
+--- DEL VariableDeclarationFragment 
+------ DEL SimpleName"
+IfStatement-6-0-0," DEL IfStatement 
+--- DEL InfixExpression 
+------ DEL SimpleName 
+------ DEL Operator 
+------ DEL NullLiteral 
+--- DEL ExpressionStatement"
+ExpressionStatement-2-0-2," INS ExpressionStatement TO Block 
+--- MOV MethodInvocation TO ExpressionStatement"
+ExpressionStatement-7-1-0," UPD ExpressionStatement 
+--- UPD MethodInvocation 
+------ UPD SimpleName 
+--------- INS StringLiteral TO SimpleName 
+--------- DEL MethodInvocation 
+------------ DEL TypeLiteral 
+------------ DEL SimpleName"
+SwitchCase-2-0-0," INS SwitchCase TO SwitchStatement 
+--- INS SimpleName TO SwitchCase"
+MethodDeclaration-2-0-1," UPD MethodDeclaration 
+--- DEL Modifier"
+ReturnStatement-2-0-0," DEL ReturnStatement 
+--- DEL SimpleName"
+TypeDeclaration-5-0-0," INS TypeDeclaration TO TypeDeclaration 
+--- INS Modifier TO TypeDeclaration 
+--- INS Modifier TO TypeDeclaration 
+--- INS SimpleName TO TypeDeclaration 
+--- INS SimpleType TO TypeDeclaration"
+ReturnStatement-2-0-1," INS ReturnStatement TO MethodDeclaration 
+--- INS SimpleName TO ReturnStatement"
+FieldDeclaration-2-0-1," UPD FieldDeclaration 
+--- DEL Modifier"
+FieldDeclaration-5-2-0," INS FieldDeclaration TO TypeDeclaration 
+--- INS Modifier TO FieldDeclaration 
+--- INS PrimitiveType TO FieldDeclaration 
+--- INS VariableDeclarationFragment TO FieldDeclaration 
+------ INS SimpleName TO VariableDeclarationFragment"
+ReturnStatement-2-0-3," UPD ReturnStatement 
+--- UPD SimpleName"
+CatchClause-3-0-0," UPD CatchClause 
+--- UPD SingleVariableDeclaration 
+------ UPD SimpleType"
+MethodDeclaration-14-0-0," INS MethodDeclaration TO TypeDeclaration 
+--- INS Modifier TO MethodDeclaration 
+--- INS PrimitiveType TO MethodDeclaration 
+--- INS SimpleName TO MethodDeclaration 
+--- INS SingleVariableDeclaration TO MethodDeclaration 
+------ INS SimpleType TO SingleVariableDeclaration 
+------ INS SimpleName TO SingleVariableDeclaration 
+--- INS ExpressionStatement TO MethodDeclaration 
+------ INS Assignment TO ExpressionStatement 
+--------- INS FieldAccess TO Assignment 
+------------ INS ThisExpression TO FieldAccess 
+------------ INS SimpleName TO FieldAccess 
+--------- INS Operator TO Assignment 
+--------- INS SimpleName TO Assignment"
+ExpressionStatement-2-0-3," INS ExpressionStatement TO MethodDeclaration 
+--- MOV MethodInvocation TO ExpressionStatement"
+FieldDeclaration-11-0-0," INS FieldDeclaration TO TypeDeclaration 
+--- INS Modifier TO FieldDeclaration 
+--- INS Modifier TO FieldDeclaration 
+--- INS Modifier TO FieldDeclaration 
+--- INS SimpleType TO FieldDeclaration 
+--- INS VariableDeclarationFragment TO FieldDeclaration 
+------ INS SimpleName TO VariableDeclarationFragment 
+------ INS MethodInvocation TO VariableDeclarationFragment 
+--------- INS SimpleName TO MethodInvocation 
+--------- INS SimpleName TO MethodInvocation 
+------------ INS TypeLiteral TO SimpleName"
+ReturnStatement-3-3-0," UPD ReturnStatement 
+--- UPD MethodInvocation 
+------ UPD QualifiedName"
+IfStatement-5-3-0," INS IfStatement TO IfStatement 
+--- MOV ExpressionStatement TO IfStatement 
+--- INS MethodInvocation TO IfStatement 
+------ INS SimpleName TO MethodInvocation 
+------ INS SimpleName TO MethodInvocation"
+VariableDeclarationStatement-7-0-2," INS VariableDeclarationStatement TO IfStatement 
+--- INS SimpleType TO VariableDeclarationStatement 
+--- INS VariableDeclarationFragment TO VariableDeclarationStatement 
+------ INS SimpleName TO VariableDeclarationFragment 
+------ INS MethodInvocation TO VariableDeclarationFragment 
+--------- INS SimpleName TO MethodInvocation 
+--------- INS SimpleName TO MethodInvocation"
+ReturnStatement-2-0-2," UPD ReturnStatement 
+--- INS SimpleName TO ReturnStatement"
+TypeDeclaration-2-1-1," UPD TypeDeclaration 
+--- INS SimpleType TO TypeDeclaration"
+MethodDeclaration-6-2-0," INS MethodDeclaration TO TypeDeclaration 
+--- INS Modifier TO MethodDeclaration 
+--- INS PrimitiveType TO MethodDeclaration 
+--- INS SimpleName TO MethodDeclaration 
+--- INS ReturnStatement TO MethodDeclaration 
+------ INS SimpleName TO ReturnStatement"
+ReturnStatement-4-2-0," UPD ReturnStatement 
+--- UPD MethodInvocation 
+------ UPD SimpleName 
+--------- INS SimpleName TO SimpleName"
+VariableDeclarationStatement-7-0-3," DEL VariableDeclarationStatement 
+--- DEL SimpleType 
+--- DEL VariableDeclarationFragment 
+------ DEL SimpleName 
+------ DEL MethodInvocation 
+--------- DEL SimpleName 
+--------- DEL SimpleName"
+ThrowStatement-4-0-0," UPD ThrowStatement 
+--- UPD ClassInstanceCreation 
+------ UPD InfixExpression 
+--------- UPD StringLiteral"
+MethodDeclaration-2-0-2," UPD MethodDeclaration 
+--- UPD Modifier"
+ExpressionStatement-5-6-1," DEL ExpressionStatement 
+--- DEL Assignment 
+------ DEL SimpleName 
+------ DEL Operator 
+------ DEL BooleanLiteral"
+VariableDeclarationStatement-4-0-1," INS VariableDeclarationStatement TO MethodDeclaration 
+--- INS SimpleType TO VariableDeclarationStatement 
+--- INS VariableDeclarationFragment TO VariableDeclarationStatement 
+------ INS SimpleName TO VariableDeclarationFragment"
+VariableDeclarationStatement-11-0-0," INS VariableDeclarationStatement TO MethodDeclaration 
+--- INS ParameterizedType TO VariableDeclarationStatement 
+------ INS SimpleType TO ParameterizedType 
+------ INS SimpleType TO ParameterizedType 
+--- INS VariableDeclarationFragment TO VariableDeclarationStatement 
+------ INS SimpleName TO VariableDeclarationFragment 
+------ INS ClassInstanceCreation TO VariableDeclarationFragment 
+--------- INS New TO ClassInstanceCreation 
+--------- INS ParameterizedType TO ClassInstanceCreation 
+------------ INS SimpleType TO ParameterizedType 
+------------ INS SimpleType TO ParameterizedType"
+TypeDeclaration-4-0-0," INS TypeDeclaration TO TypeDeclaration 
+--- INS Modifier TO TypeDeclaration 
+--- INS Modifier TO TypeDeclaration 
+--- INS SimpleName TO TypeDeclaration"
+IfStatement-3-1-0," UPD IfStatement 
+--- UPD InfixExpression 
+------ UPD SimpleName"
+VariableDeclarationStatement-5-2-2," DEL VariableDeclarationStatement 
+--- DEL SimpleType 
+--- DEL VariableDeclarationFragment 
+------ DEL SimpleName 
+------ DEL NullLiteral"
+ThrowStatement-3-1-0," UPD ThrowStatement 
+--- UPD ClassInstanceCreation 
+------ UPD SimpleType"
+MethodDeclaration-4-1-0," UPD MethodDeclaration 
+--- INS SingleVariableDeclaration TO MethodDeclaration 
+------ INS PrimitiveType TO SingleVariableDeclaration 
+------ INS SimpleName TO SingleVariableDeclaration"
+VariableDeclarationStatement-7-0-6," INS VariableDeclarationStatement TO EnhancedForStatement 
+--- INS SimpleType TO VariableDeclarationStatement 
+--- INS VariableDeclarationFragment TO VariableDeclarationStatement 
+------ INS SimpleName TO VariableDeclarationFragment 
+------ INS MethodInvocation TO VariableDeclarationFragment 
+--------- INS SimpleName TO MethodInvocation 
+--------- INS SimpleName TO MethodInvocation"
+FieldDeclaration-9-0-1," INS FieldDeclaration TO TypeDeclaration 
+--- INS Modifier TO FieldDeclaration 
+--- INS Modifier TO FieldDeclaration 
+--- INS SimpleType TO FieldDeclaration 
+--- INS VariableDeclarationFragment TO FieldDeclaration 
+------ INS SimpleName TO VariableDeclarationFragment 
+------ INS ClassInstanceCreation TO VariableDeclarationFragment 
+--------- INS New TO ClassInstanceCreation 
+--------- INS SimpleType TO ClassInstanceCreation"

python/addNewData.py

@@ -0,0 +1,142 @@
+from common.commons import *
+DATA_PATH = os.environ["DATA_PATH"]
+
+def core():
+    clusterPath = join(DATA_PATH, 'shapes')
+    roots = listdir(clusterPath)
+    roots = [i for i in roots if not (i.startswith('.') or i.endswith('.pickle'))]
+    pattern = {}
+    for root in roots:
+        root
+        sizes = listdir(join(clusterPath, root))
+        for size in sizes:
+            # actions = listdir(join(clusterPath,root,size))
+            # for action in actions:
+            clusters = listdir(join(clusterPath, root, size))
+            for cluster in clusters:
+                members = listdir(join(clusterPath, root, size, cluster))
+
+                # pattern[root+'/'+size+'/'+cluster]= root +'/' +size +'/'+ members[0]
+                pattern[root+'/'+size+'/'+cluster]=  members[0]
+    pattern
+
+    from pairs import shapePairs
+    matches = shapePairs()
+    # 'FFmpeg','curl','nginx','openssl','redis','tmux','vlc']
+    matches = matches[matches.file.apply(lambda x: x in list(pattern.values()) or not ( x.startswith('linux_') or  x.startswith('FFmpeg_') or x.startswith('curl_') or  x.startswith('nginx_') or  x.startswith('openssl_') or  x.startswith('redis_') or x.startswith('tmux_') or  x.startswith('vlc_')))]
+    from pairs import createPairs
+    createPairs(matches)
+    # # # elif job == 'importShapesPairs':
+    from pairs import importShape
+    importShape()
+
+def checkWrongMembers():
+    clusterPath = join(DATA_PATH, 'shapes')
+    roots = listdir(clusterPath)
+    roots = [i for i in roots if not (i.startswith('.') or i.endswith('.pickle'))]
+    pattern = {}
+    for root in roots:
+        root
+        sizes = listdir(join(clusterPath, root))
+        for size in sizes:
+            # actions = listdir(join(clusterPath,root,size))
+            # for action in actions:
+            clusters = listdir(join(clusterPath, root, size))
+            for cluster in clusters:
+                members = listdir(join(clusterPath, root, size, cluster))
+                sizeDict = {}
+                for s in [(i,os.path.getsize(join(clusterPath, root, size, cluster,i))) for i in members]:
+                    sizeDict[s[1]] = s[0]
+                sizeDict
+                if len(sizeDict) > 1:
+                    print(join(clusterPath, root, size, cluster))
+                    print(sizeDict.values())
+
+def cluster():
+    clusterPath = join(DATA_PATH, 'shapes')
+    roots = listdir(clusterPath)
+    roots = [i for i in roots if not (i.startswith('.') or i.endswith('.pickle'))]
+    pattern = {}
+    for root in roots:
+        root
+        sizes = listdir(join(clusterPath, root))
+        for size in sizes:
+            # actions = listdir(join(clusterPath,root,size))
+            # for action in actions:
+            clusters = listdir(join(clusterPath, root, size))
+            for cluster in clusters:
+                members = listdir(join(clusterPath, root, size, cluster))
+
+                # pattern[root+'/'+size+'/'+cluster]= root +'/' +size +'/'+ members[0]
+                pattern[root+'/'+size+'/'+cluster]=  members[0]
+    pattern
+
+    pairsPath = join(DATA_PATH, 'pairs')
+    from abstractPatch import loadPairMulti
+    for root in roots:
+        matches =loadPairMulti(root,'','shapes')
+        matches
+        sizes = matches['sizes'].unique().tolist()
+        for s in sizes:
+            match = matches[matches['sizes'] == s]
+            match
+            clusterCore(pattern,clusterPath, 'shapes', match, pairsPath, root, s, '')
+
+def clusterCore(pattern,clusterPath, level, match, pairsPath, root, s,action ,token=''):
+    col_combi = match.tuples.values.tolist()
+    import networkx
+    g = networkx.Graph(col_combi)
+    cluster = []
+    for subgraph in networkx.connected_component_subgraphs(g):
+        logging.info('Cluster size %d',len(subgraph.nodes()))
+        cluster.append(subgraph.nodes())
+    cluster
+    pathMapping = dict()
+    if level == 'actions':
+        indexFile = join(pairsPath, root, s,action+'.index')
+    elif level == 'shapes':
+        indexFile = join(pairsPath, root, s + '.index')
+    else:
+        indexFile =join(pairsPath, root, s,action,token+'.index')
+    df = pd.read_csv(indexFile, header=None, usecols=[0, 1], index_col=[0])
+    pathMapping = df.to_dict()
+
+    workList = []
+
+    exportCLusters ={}
+    if not os.path.exists(join(clusterPath, root, s)):
+        print()
+        existingClusters = 0
+    else:
+        existingClusters = len(listdir(join(clusterPath, root, s)))
+
+    for clus in cluster:
+        members = [pathMapping[1][int(i)] for i in clus]
+        members
+        potentialClusters = [(key, value) for key, value in pattern.items() if key.startswith(root + '/' + s)]
+        potentialClusters
+        foundExisting = False
+        for pc,pcMember in potentialClusters:
+            if pcMember in members:
+                pc
+                foundExisting = True
+                exportCLusters[pc.split('/')[-1]] = members
+        if not foundExisting:
+            exportCLusters[existingClusters] = members
+            existingClusters= existingClusters+1
+    exportCLusters
+    for k,v in exportCLusters.items():
+        for f in v:
+            t = f, root, level, clusterPath, s, action, token, k
+            workList.append(t)
+    # for idx, clus in enumerate(cluster):
+    #     logging.info('exporting cluster %s %s %s %d', root,s,action,idx)
+    #     for f in clus:
+    #         dumpFile = pathMapping[1][int(f)]
+    #
+    #         t = dumpFile,root,level,clusterPath,s,action,token,idx
+    #         workList.append(t)
+    from abstractPatch import dumpFilesCore
+    parallelRun(dumpFilesCore,workList)
+    # for wl in workList:
+    #     dumpFilesCore(wl)

python/bugReportDownloader.py

@@ -0,0 +1,132 @@
+from common.commons import *
+
+BUG_REPORT_PATH = os.environ["BUG_REPORT"]
+DATA_PATH = os.environ["DATA_PATH"]
+# BUG_POINT = os.environ["BUG_POINT"]
+COMMIT_DFS = os.environ["COMMIT_DFS"]
+from urllib.request import urlopen
+from urllib import error
+import urllib
+
+import socket
+
+timeout = 30
+socket.setdefaulttimeout(timeout)
+
+import logging
+hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
+       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+       'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
+       'Accept-Encoding': 'none',
+       'Accept-Language': 'en-US,en;q=0.8',
+       'Connection': 'keep-alive'}
+
+def bugRepoDict():
+    brDict={}
+    brDict['CAMEL'] ='https://issues.apache.org/jira/browse/CAMEL-'#4748'
+    brDict['HBASE'] ='https://issues.apache.org/jira/browse/HBASE-'#84'
+    brDict['HIVE'] ='https://issues.apache.org/jira/browse/HIVE-'#3518'
+    brDict['CODEC'] ='https://issues.apache.org/jira/browse/CODEC-'#103'
+    brDict['COLLECTIONS'] ='https://issues.apache.org/jira/browse/COLLECTIONS-'#660'
+    brDict['COMPRESS'] ='https://issues.apache.org/jira/browse/COMPRESS-'#261'
+    brDict['CONFIGURATION'] ='https://issues.apache.org/jira/browse/CONFIGURATION-'#307'
+    brDict['CRYPTO'] ='https://issues.apache.org/jira/browse/CRYPTO-'#85'
+    brDict['CSV'] ='https://issues.apache.org/jira/browse/CSV-'#84'
+    brDict['IO'] ='https://issues.apache.org/jira/browse/IO-'#193'
+    brDict['LANG'] ='https://issues.apache.org/jira/browse/LANG-'#810'
+    brDict['MATH'] ='https://issues.apache.org/jira/browse/MATH-'#790'
+    brDict['WEAVER'] = 'https://issues.apache.org/jira/browse/WEAVER-'
+    brDict['ENTESB'] ='https://issues.jboss.org/browse/ENTESB-'#80'
+    brDict['JBMETA'] ='https://issues.jboss.org/browse/JBMETA-'#315'
+    brDict['ELY'] ='https://issues.jboss.org/browse/ELY-'#515'
+    brDict['SWARM'] ='https://issues.jboss.org/browse/SWARM-'#1710'
+    brDict['WFARQ'] ='https://issues.jboss.org/browse/WFARQ-'#14'
+    brDict['WFCORE'] ='https://issues.jboss.org/browse/WFCORE-'#1499'
+    brDict['WFLY'] ='https://issues.jboss.org/browse/WFLY-'#3717'
+    brDict['WFMP'] ='https://issues.jboss.org/browse/WFMP-'#85'
+    brDict['AMQP'] ='https://jira.spring.io/browse/AMQP-'#32'
+    brDict['ANDROID'] ='https://jira.spring.io/browse/ANDROID-'#131'
+    brDict['BATCH'] ='https://jira.spring.io/browse/BATCH-'#84'
+    brDict['BATCHADM'] ='https://jira.spring.io/browse/BATCHADM-'#22'
+    brDict['DATACMNS'] ='https://jira.spring.io/browse/DATACMNS-'#43'
+    brDict['DATAGRAPH'] ='https://jira.spring.io/browse/DATAGRAPH-'#869'
+    brDict['DATAJPA'] ='https://jira.spring.io/browse/DATAJPA-'#869'
+    brDict['DATAJPA'] ='https://jira.spring.io/browse/DATAJPA-'#43'
+    brDict['DATAMONGO'] ='https://jira.spring.io/browse/DATAMONGO-'#634'
+    brDict['DATAREDIS'] ='https://jira.spring.io/browse/DATAREDIS-'#680'
+    brDict['DATAREST']='https://jira.spring.io/browse/DATAREST-'#44'
+    brDict['LDAP'] ='https://jira.spring.io/browse/LDAP-'#64' #https://github.com/spring-projects/spring-ldap/issues/107
+    brDict['MOBILE'] = 'https://jira.spring.io/browse/MOBILE-'
+    brDict['ROO'] ='https://jira.spring.io/browse/ROO-'#260'
+    brDict['SEC'] = 'https://jira.spring.io/browse/SEC-'#1880' #'https://github.com/spring-projects/spring-security/issues/2108'
+    brDict['SECOAUTH'] ='https://jira.spring.io/browse/SECOAUTH-'#42'
+    brDict['SGF'] ='https://jira.spring.io/browse/SGF-'#69'
+    brDict['SHDP'] ='https://jira.spring.io/browse/SHDP-'#444'
+    brDict['SHL'] ='https://jira.spring.io/browse/SHL-'#80'
+    brDict['SOCIAL'] ='https://jira.spring.io/browse/SOCIAL-'#33'
+    brDict['SOCIALFB'] ='https://jira.spring.io/browse/SOCIALFB-'#33'
+    brDict['SOCIALLI'] ='https://jira.spring.io/browse/SOCIALLI-'#33'
+    brDict['SOCIALTW'] ='https://jira.spring.io/browse/SOCIALTW-'#10'
+    brDict['SPR'] ='https://jira.spring.io/browse/SPR-'#6132'
+    brDict['SWF'] ='https://jira.spring.io/browse/SWF-'#80'
+    brDict['SWS'] ='https://jira.spring.io/browse/SWS-'#510'
+    brDict['AspectJ']='https://bugs.eclipse.org/bugs/show_bug.cgi?id='
+    brDict['JDT'] ='https://bugs.eclipse.org/bugs/show_bug.cgi?id='#30113'
+    brDict['SWT'] ='https://bugs.eclipse.org/bugs/show_bug.cgi?id='#231787'
+    brDict['PDE'] ='https://bugs.eclipse.org/bugs/show_bug.cgi?id='#201369'
+    return brDict
+
+def downloadAll(x):
+    try:
+        pj,id = x.split('-')
+        links = bugRepoDict()
+        downloadLink = links[pj] + id
+        webRequest(downloadLink)
+
+    except Exception as e:
+        print(e)
+        logging.error(e)
+        return False
+def webRequest(x):
+    url = x
+    bugID = url.split('/')[-1:]
+    url = url + '?redirect=false'
+
+    brLocation = join(BUG_REPORT_PATH,bugID[0] + ".xml")
+    if isfile(brLocation):
+        with open(brLocation, 'rb') as f:
+            the_page = p.load(f)
+    else:
+        try:
+            logging.info(url)
+            req = urllib.request.Request(url, headers=hdr)
+
+
+            response = urlopen(req)
+            the_page = response.read()
+        except error.HTTPError as err:
+            if err.code == 404:
+                print("Error: %s, reason: %s." % (err.code, err.reason))
+            return None
+        p.dump(the_page, open(brLocation, "wb"))
+
+def caseBRDownload(subject):
+
+    if not os.path.exists(BUG_REPORT_PATH):
+        os.mkdir(BUG_REPORT_PATH)
+    bids=[]
+    if subject == 'ALL':
+        for i in listdir(COMMIT_DFS):
+            commits = load_zipped_pickle(join(COMMIT_DFS, i))
+            bids.extend(commits.fix.values.tolist())
+    else:
+        subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+
+        subjects = subjects.query("Subject == '{0}'".format(subject))
+
+        commits = load_zipped_pickle(join(COMMIT_DFS, subjects.iloc[0].Repo + '.pickle'))
+        bids.extend(commits.fix.values.tolist())
+
+
+
+    parallelRun(downloadAll,bids)

python/bugReportParser.py

@@ -0,0 +1,210 @@
+from common.commons import *
+
+# CODE_PATH = os.environ["CODE_PATH"]
+DATA_PATH = os.environ["DATA_PATH"]
+BUG_REPORT_PATH = os.environ["BUG_REPORT"]
+COMMIT_DFS = os.environ["COMMIT_DFS"]
+
+import bs4 as bs
+
+import re
+def parseCore(br):
+    columns = ['bugReport', 'summary', 'description','created','updated','resolved','reporterDN','reporterEmail','hasAttachment','attachmentTime','hasPR','commentsCount']
+    bugReport = pd.DataFrame(columns=columns)
+    ind = 0
+    if isfile(join(BUG_REPORT_PATH, br)):
+        
+        with open(join(BUG_REPORT_PATH, br), 'rb') as f:
+            the_page = p.load(f)
+        soup = bs.BeautifulSoup(the_page, "html.parser")
+
+        if not (br.startswith('show')):
+
+            type = soup.find('span', {'id': 'type-val'})
+            status = soup.find('span', {'id': 'status-val'})
+
+            if type is None:
+                # print(br)
+                return;
+            if (type.text.strip() == 'Bug' or type.text.strip() =='Defect'):
+                if (status.text.strip() == 'Resolved' or status.text.strip() == 'Closed'):
+
+                    summary = soup.find('h1', {'id': 'summary-val'}).text
+                    desc = soup.find('div', {'id': 'description-val'})
+
+                    created= soup.find('span', {'data-name': 'Created'}).time['datetime']
+                    updated = soup.find('span', {'data-name': 'Updated'}).time['datetime']
+                    resolved = soup.find('span', {'data-name': 'Resolved'}).time['datetime']
+
+                    # jboss reporter is different
+                    reporterField = soup.find('span',{'id':'reporter-val'})
+                    try:
+                        reporterInfo= reporterField.findAll('span', {'class': 'user-hover'})[0]['data-user']
+                        reporterDict = eval(reporterInfo)
+                        reporterDN = reporterDict['displayName']
+                        reporterEmail = reporterDict['emailAddress']
+                    except KeyError as e:
+
+                        e
+                        reporterDN = reporterField.text.strip()
+                        reporterEmail = None
+                    except Exception as e:
+                        e
+                        reporterDN = reporterField.text.strip()
+                        reporterEmail = None
+
+                    attachment = soup.find('dd', {'class': 'attachment-date'})
+                    isAttachment = False
+                    isPR = False
+                    attachmentTime = None
+                    if attachment is not None and len(attachment) > 0:
+                        attachmentTime = attachment.parent.time['datetime']
+                        isAttachment = True
+
+                    hasPullReq = soup.find('strong', {'title': 'Git Pull Request'})
+                    if hasPullReq is not None:
+                        haveAttachment = hasPullReq.parent.findAll('a')
+                        if(haveAttachment is not None and len(haveAttachment)>0):
+                            isPR = True
+                        else:
+                            linkPullRequest = soup.find('ul', {'id': 'issuedetails'}).find('a', {'title': 'PullRequest'})
+                            if(linkPullRequest is not None and len(linkPullRequest)> 0):
+                                isPR = True
+
+                    commentsCount = len(re.findall(r'\"comment-[0-9]+', soup.getText()))
+                    if desc is not None:
+                        # bugReport.loc[ind] = [br, summary, desc.text,created,updated,resolved,reporterDN,reporterEmail,isAttachment,attachmentTime,isPR,commentsCount]
+                        return [br, summary, desc.text, created, updated, resolved, reporterDN,
+                                              reporterEmail, isAttachment, attachmentTime, isPR, commentsCount]
+                        # ind += 1
+                    else:
+                        # bugReport.loc[ind] = [br, summary, None, created, updated, resolved, reporterDN, reporterEmail,
+                        #                       isAttachment,attachmentTime,isPR,commentsCount]
+                        return [br, summary, None, created, updated, resolved, reporterDN, reporterEmail,
+                                              isAttachment,attachmentTime,isPR,commentsCount]
+                        # ind += 1
+            # else:
+            #     print(type.text.strip())
+        else:
+            importance = (
+                soup.find('a', {'href': 'page.cgi?id=fields.html#importance'}).parent.parent.parent.td.text.strip())
+            status = soup.find('span', {'id': 'static_bug_status'}).text.strip()
+
+            m1 = re.search('enhancement', importance, re.IGNORECASE)
+            # if not m:
+            #     print(br)
+            if not m1:
+                m = re.search('FIXED|DUPLICATE', status)
+                if m:
+                    summary = soup.find('span', {'id': 'short_desc_nonedit_display'}).text
+                    comment0 = soup.find('div', {'id': 'c0'})
+                    if comment0.find('a').text == 'Description':
+                        desc = comment0.find('pre', {'class': 'bz_comment_text'}).text
+                        bugReport.loc[ind] = [br, summary, desc]
+                        ind += 1
+                # desc
+        # return  bugReport
+
+def parallelRun(coreFun,elements,*args):
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        try:
+            dataL = []
+            futures = {executor.submit(coreFun, l,*args): l for l in elements}
+            for future in concurrent.futures.as_completed(futures):
+                url = futures[future]
+                try:
+                    data = future.result()
+                    dataL.append(data)
+
+                except Exception as exc:
+                    logging.error('%r generated an exception: %s' % (url, exc))
+                    raise
+
+                kwargs = {
+                    'total': len(futures),
+                    'unit': 'files',
+                    'unit_scale': True,
+                    'leave': False
+                }
+                # Print out the progress as tasks complete
+                for f in tqdm(concurrent.futures.as_completed(futures), **kwargs):
+                    pass
+            newData = pd.concat(dataL)
+            return newData
+        except Exception as e:
+            executor.shutdown()
+            raise
+
+def getCommitter(x):
+    subject = x.split('-')[0]
+    subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+    repo = subjects.query("Subject == '{0}'".format(subject)).Repo.tolist()[0]
+    commits = load_zipped_pickle(join(COMMIT_DFS, repo+'.pickle'))
+    commits
+    committer = commits.query("fix =='{0}'".format(x)).committer.tolist()
+    # print(len(committer))
+    return committer
+
+
+
+def step1(subject):
+
+    if subject == 'ALL':
+        bids = os.listdir(BUG_REPORT_PATH)
+    else:
+        bids = [f for f in os.listdir(BUG_REPORT_PATH) if f.startswith(subject)]
+    bids = [i for i in bids if not i.startswith('.')]
+
+    # bids = bids[:100]
+
+
+
+    dataL = parallelRunMerge(parseCore,bids)
+    logging.info('Finish parsing')
+    # list(filter(None.__ne__, dataL))
+    br = pd.DataFrame(
+        columns=['bugReport', 'summary', 'description', 'created', 'updated', 'resolved', 'reporterDN', 'reporterEmail',
+                 'hasAttachment', 'attachmentTime', 'hasPR', 'commentsCount'], data=list(filter(None.__ne__, dataL)))
+    # br = pd.concat(dataL)
+    logging.info('Finish parsing')
+    br['project'] = br.bugReport.apply(lambda x: x.split('-')[0])
+    br['bid'] = br.bugReport.apply(lambda x: x.split('.')[0])
+    br['committerEmail'] = br.bid.apply(lambda x: getCommitter(x))
+
+    br['sameEmail'] = br.apply(lambda x:x['reporterEmail'] in x['committerEmail'],axis=1)
+    subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+    subjects['Group']=subjects.Group.apply(lambda x:x.replace('Commons','Apache').replace('Wildfly','Jboss').upper())
+    save_zipped_pickle(br,join(DATA_PATH,subject+"bugReportsComplete.pickle"))
+
+
+def step3(subject):
+    subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+    subjects['Group']=subjects.Group.apply(lambda x:x.replace('Commons','Apache').replace('Wildfly','Jboss').upper())
+
+    stats = load_zipped_pickle(join(DATA_PATH, subject + "bugReportsComplete.pickle"))
+
+    stats = stats[['bugReport', 'sameEmail', 'hasAttachment','attachmentTime','hasPR', 'created','codeElements','stackTraces','summaryHints','descHints','commentsCount']]
+    stats['project'] =stats.bugReport.apply(lambda x:x.split('-')[0])
+    stats['project'] = stats.project.apply(lambda x: subjects.query("Subject == '{0}'".format(x)).Group.tolist()[0]+'-'+x)
+    stats['created'] = stats['created'].apply(lambda x: pd.to_datetime(x))
+    stats['attachmentTime'] = stats['attachmentTime'].apply(lambda x: pd.to_datetime(x))
+    stats['isAttach'] = (stats['attachmentTime'] - stats['created']) < pd.Timedelta(1, unit='h')
+    stats['hints'] = stats.apply(lambda x: True if len(x['summaryHints']) > 0 or len(x['descHints']) > 0 else False,
+                                 axis=1)
+
+    stats['commentsCount'].to_csv(subject+'Comments', index=False)
+
+    a = stats.groupby('project').agg({'sameEmail': "sum", 'isAttach': "sum"})
+    a['avgComments'] =stats['commentsCount'].mean()
+    a['hasStackTraces'] = len([i for i in stats['stackTraces'].str.len().tolist() if i != 0])
+    a['hasHints']= len([i for i in stats['hints'].tolist() if i == True])
+    a['hasCE']= len([i for i in stats['codeElements'].str.len().tolist() if i != 0])
+
+    a['no'] = len(stats)
+
+    a.to_latex(join(DATA_PATH,subject+'datasetQuality.tex'))
+
+def caseBRParser(subject):
+    step1(subject)
+    # step2()
+    # step3()

python/bugstats.py

@@ -0,0 +1,233 @@
+from common.commons import *
+DATA_PATH = os.environ["DATA_PATH"]
+PROJECT_TYPE = os.environ["PROJECT_TYPE"]
+
+
+def bStats():
+    if isfile(join(DATA_PATH, 'studyBugReports.pickle')):
+        studyBugReports = load_zipped_pickle(join(DATA_PATH, 'studyBugReports.pickle'))
+    else:
+        brs = load_zipped_pickle(join(DATA_PATH, args.subject + "bugReportsComplete.pickle"))
+        commits = load_zipped_pickle(join(DATA_PATH, 'singleBR.pickle'))
+
+        dbDir = join(DATA_PATH, 'redis')
+
+        portInner = '6399'
+        startDB(dbDir, portInner, PROJECT_TYPE)
+
+        import redis
+
+        redis_db = redis.StrictRedis(host="localhost", port=portInner, db=0)
+        keys = redis_db.scan(0, match='*', count='1000000')
+
+        matches = pd.DataFrame(keys[1], columns=['pairs_key'])
+
+        # matches = load_zipped_pickle(join(DATA_PATH,'singleHunks'))
+        matches['pairs_key'] = matches['pairs_key'].apply(lambda x: x.decode())
+        matches['root'] = matches['pairs_key'].apply(lambda x: x.split('/')[0])
+        matches['size'] = matches['pairs_key'].apply(lambda x: x.split('/')[1])
+        matches['file'] = matches['pairs_key'].apply(lambda x: x.split('/')[2])
+        matches['repo'] = matches['file'].apply(lambda x: x.split('_')[0])
+        matches['commit'] = matches['file'].apply(lambda x: x.split('_')[1])
+
+        subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+
+        def getBID(x):
+            try:
+                if x['repo'].endswith('.git'):
+                    return None
+                subject = subjects.query("Repo == '{0}'".format(x['repo'])).Subject.tolist()[0]
+                bids = commits.query(
+                    "commit.str.startswith('{0}') and project== '{1}'".format(x['commit'], subject)).bid.tolist()
+                return bids[0]
+            except Exception as e:
+                logging.error(e)
+
+        matches = matches[~matches.repo.apply(
+            lambda i: (i.startswith('commons-math') or i.startswith('commons-lang') or i.startswith(
+                'closure-compiler') or i.startswith('joda-time') or i.startswith('mockito') or i.startswith(
+                'jfreechart')))]
+        matches['bid'] = matches.apply(lambda x: getBID(x), axis=1)
+
+        subjects
+        # res = pd.merge(matches, brs, on=['bid'])
+        save_zipped_pickle(matches, join(DATA_PATH, 'studyDataset.pickle'))
+        studyBugReports = brs[brs.bid.isin(matches.bid.unique())]
+        save_zipped_pickle(studyBugReports, join(DATA_PATH, 'studyBugReports.pickle'))
+    if isfile(join(DATA_PATH, 'studyBR_DTM_index')):
+        brIndexes = load_zipped_pickle(join(DATA_PATH, 'studyBR_DTM_index'))
+        bugDTM = load_zipped_pickle(join(DATA_PATH, 'studyBR_DTM'))
+        vectorDF = load_zipped_pickle(join(DATA_PATH, 'studyBR_vector'))
+        matches = load_zipped_pickle(join(DATA_PATH, 'studyDataset.pickle'))
+    else:
+        studyBugReports['description'] = studyBugReports['description'].fillna("")
+        studyBugReports['sumDesc'] = studyBugReports['summary'] + studyBugReports['description']
+        # corpus['sumDesc'] = corpus['summary'] + corpus['desc']
+        # from common.preprocessing import
+        # result, aVector = getVectorAndDtm(corpus, 'summary')
+        from common.preprocessing import calculateTfIdfNLList
+
+        corpusBug = studyBugReports['sumDesc'].values.tolist()
+        from common.preprocessing import preprocessingNL
+
+        preCorpusBug = list(map(preprocessingNL, corpusBug))
+
+        v = calculateTfIdfNLList(preCorpusBug)
+        bugDTM = v.transform(preCorpusBug)
+        bugDTM
+        save_zipped_pickle(bugDTM, join(DATA_PATH, 'studyBR_DTM'))
+        brIndexes = studyBugReports['bid'].values.tolist()
+
+        save_zipped_pickle(brIndexes, join(DATA_PATH, 'studyBR_DTM_index'))
+        # from sklearn.metrics.pairwise import cosine_similarity
+        # cosine_similarity(bugDTM[11701], bugDTM[11111])
+        vectorDF = pd.DataFrame(columns=['bid', 'dtm'])
+        # idx = 0
+        for idx, val in enumerate(brIndexes):
+            vectorDF.loc[idx] = [val, bugDTM[idx]]
+        vectorDF
+
+        save_zipped_pickle(vectorDF, join(DATA_PATH, 'studyBR_vector'))
+
+    matches
+    if isfile(join(DATA_PATH, 'study_clusters')):
+        clustersDF = load_zipped_pickle(join(DATA_PATH, 'study_clusters'))
+    else:
+        clustersDF = pd.DataFrame(columns=['cid', 'type', 'members'])
+        idx = 0
+
+        def statsCore(cs, type):
+            global idx
+
+            cs = [i for i in cs if not (i.startswith('commons-math') or i.startswith('commons-lang') or i.startswith(
+                'closure-compiler') or i.startswith('joda-time') or i.startswith('mockito') or i.startswith(
+                'jfreechart'))]
+            # print('Cluster %s : member size %s' % (shape+"-"+size +"-"+cluster, len(cs)))
+            if len(cs) > 0:
+                if token is None:
+                    if action is None:
+                        t = shape + "-" + size + "-" + cluster
+
+                        clustersDF.loc[idx] = [t, type, cs]
+                        idx = idx + 1
+                    else:
+                        t = shape + "-" + size + "-" + cluster + "-" + action  # , len(cs)
+                        clustersDF.loc[idx] = [t, type, cs]
+                        idx = idx + 1
+                else:
+                    # clusterSize = len(cs)
+                    # if clusterSize > 0:
+                    #     clusterSize = len(set([re.split('.txt_[0-9]+', i)[0] for i in cs]))
+                    t = shape + "-" + size + "-" + cluster + "-" + action + "-" + token  # , clusterSize
+                    clustersDF.loc[idx] = [t, type, cs]
+                    idx = idx + 1
+
+        for type in ['tokens', 'actions', 'shapes']:
+            shapesPath = join(DATA_PATH, type)
+            shapes = listdir(shapesPath)
+            shapes = [f for f in shapes if isdir(join(shapesPath, f))]
+            shape = size = cluster = action = token = None
+
+            for shape in shapes:
+                if shape.startswith('.'):
+                    continue
+                sizes = listdir(join(shapesPath, shape))
+
+                for size in sizes:
+                    if size.startswith('.'):
+                        continue
+                    clusters = listdir(join(shapesPath, shape, size))
+                    for cluster in clusters:
+                        if cluster.startswith('.'):
+                            continue
+                        cs = listdir(join(shapesPath, shape, size, cluster))
+
+                        if shapesPath.endswith('shapes'):
+                            cs = listdir(join(shapesPath, shape, size, cluster))
+                            statsCore(cs, 'shapes')
+                        else:
+                            # level3
+                            for action in cs:
+                                if action.startswith('.'):
+                                    continue
+                                tokens = listdir(join(shapesPath, shape, size, cluster, action))
+                                if shapesPath.endswith('actions'):
+                                    statsCore(tokens, 'actions')
+                                else:
+                                    for token in tokens:
+                                        if token.startswith('.'):
+                                            continue
+                                        cs = listdir(join(shapesPath, shape, size, cluster, action, token))
+                                        statsCore(cs, 'tokens')
+
+        clustersDF
+        save_zipped_pickle(clustersDF, join(DATA_PATH, 'study_clusters'))
+        clustersDF
+
+        # selected = clustersDF[clustersDF.type =='shapes']
+
+        from sklearn.metrics.pairwise import cosine_similarity
+        # cosine_similarity(bugDTM[11701], bugDTM[11111])
+        def getSimilarity(x):
+            try:
+                if len(x) == 1:
+                    return [1]
+                else:
+                    filenames = list(set([re.split('.txt_[0-9]+', i)[0] for i in x]))
+                    if len(filenames) == 1:
+                        return [1]
+                    else:
+                        bids2Compare = [matches[matches.file.str.startswith(fn)].bid.unique()[0] for fn in filenames]
+
+                        pairs = list(itertools.combinations(bids2Compare, 2))
+                        pairs
+                        res = []
+                        for p in pairs:
+                            p
+                            simi = cosine_similarity(vectorDF[vectorDF.bid == p[0]].iloc[0].dtm,
+                                                     vectorDF[vectorDF.bid == p[1]].iloc[0].dtm)
+                            res.append(simi[0][0])
+                        return res
+            except Exception as e:
+                logging.error(e)
+
+        # import swifter
+        clustersDF['simi'] = clustersDF.members.apply(lambda x: getSimilarity(x))
+        save_zipped_pickle(clustersDF, join(DATA_PATH, 'study_clusters'))
+
+    clustersDF
+
+    shapes = clustersDF[clustersDF.type == 'shapes']
+    actions = clustersDF[clustersDF.type == 'actions']
+    tokens = clustersDF[clustersDF.type == 'tokens']
+
+    # shapes
+    # yList = [list(itertools.chain.from_iterable(shapes.simi.values.tolist())),
+    # list(itertools.chain.from_iterable(actions.simi.values.tolist())),
+    # list(itertools.chain.from_iterable(tokens.simi.values.tolist()))]
+    # colNames = ['shapes','actions','tokens']
+
+    ys = []
+    cols = []
+    means = []
+    # plotBox(yList, colNames, 'bugReport' + '.pdf', True)
+    for ds in [shapes, actions, tokens]:
+        ds['ms'] = ds.members.str.len()
+        ds.sort_values(by=['ms'], ascending=False, inplace=True)
+        top10 = ds.head(20)
+
+        colNames = top10.cid.values.tolist()
+        # colNames = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
+        colNames = list(range(len(colNames)))
+        yList = yList = top10.simi.values.tolist()
+        # yList = [np.mean(i) for i in yList]
+        # colNames.insert(0,'ALL')
+        # yList.insert(0,list(itertools.chain.from_iterable(ds.simi.values.tolist())))
+        mean = np.mean(list(itertools.chain.from_iterable(ds.simi.values.tolist())))
+        type = ds.type.iloc[0]
+        # from common.commons import plotBox
+        # plotBox(yList,colNames,type+'.pdf',False)
+        ys.append(yList)
+        cols.append(colNames)
+        means.append(mean)
+    plotBox2(ys, cols, 'test.pdf', means, False)

python/commitCollector.py

@@ -0,0 +1,77 @@
+
+
+from common.commons import *
+
+REPO_PATH = os.environ["REPO_PATH"]
+DATA_PATH = os.environ["DATA_PATH"]
+COMMIT_DFS = os.environ["COMMIT_DFS"]
+COMMIT_FOLDER = os.environ["COMMIT_FOLDER"]
+
+def getCommitFromRepo(f,gitrepo,branch):
+    cmd = 'git -C ' + f + ' checkout -f ' + branch
+
+    output, err = shellGitCheckout(cmd)
+    m = re.search(branch, err)
+
+    while not m:
+        time.sleep(10)
+        logging.info('Waiting for checkout')
+    cmd = 'git -C ' + f + " log --no-merges --pretty=format:'{\"commit\":\"%H\",\"commitDate\":\"%ci\",\"title\":\"%f\",\"committer\":\"%ce\"}' > " + gitrepo + '.commits'
+    output = shellCallTemplate(cmd,enc='latin1')
+
+
+def makeDF(filename):
+    with open(filename,encoding='latin1') as f:
+        lines = f.readlines()
+    ls = [eval(f) for f in lines]
+    ds  = pd.DataFrame.from_dict(ls)
+    ds['commitDate']= ds['commitDate'].apply(lambda x:pd.to_datetime(x))
+    return ds
+
+
+def caseCollect(subject):
+    
+    if not os.path.exists(COMMIT_FOLDER):
+        os.mkdir(COMMIT_FOLDER)
+    if not os.path.exists(COMMIT_DFS):
+        os.mkdir(COMMIT_DFS)
+        
+    subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+    if subject == 'ALL':
+        tuples = subjects[['Repo', 'Branch']].values.tolist()
+    else:
+        # repos = subjects.query("Subject == '{0}'".format(subject)).Repo.tolist()
+        tuples = subjects.query("Subject == '{0}'".format(subject))[['Repo', 'Branch']].values.tolist()
+
+    for t in tuples:
+        repo,branch = t
+        logging.info(repo)
+        getCommitFromRepo(join(REPO_PATH, repo), join(COMMIT_FOLDER, repo),branch)
+
+    if subject == 'ALL':
+        commits = listdir(COMMIT_FOLDER)
+    else:
+        commits = [i for i in listdir(COMMIT_FOLDER) if i.startswith(repo)]
+
+    for commit in commits:
+        logging.info(commit)
+        rDF = makeDF(join(COMMIT_FOLDER, commit))
+        repoName = commit.split('.')[0]
+        save_zipped_pickle(rDF, join(COMMIT_DFS, repoName + ".pickle"))
+        # p.dump(rDF, open(join(COMMIT_DFS, repoName + ".pickle"), "wb"))
+
+def caseClone(subject):
+    if not os.path.exists(REPO_PATH):
+        os.mkdir(REPO_PATH)
+
+    subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+    if subject == 'ALL':
+        gitrepos = subjects.GitRepo.tolist()
+    else:
+        gitrepos = subjects.query("Subject == '{0}'".format(subject)).GitRepo.tolist()
+    os.getcwd()
+    os.chdir(REPO_PATH)
+    for gitrepo in gitrepos:
+        cmd = 'git clone ' + gitrepo
+        out = shellCallTemplate(cmd)
+

python/common/commons.py

@@ -0,0 +1,748 @@
+
+import logging
+import sys
+import gzip
+import numpy as np
+from tqdm import tqdm
+import shutil
+import itertools
+
+# COMMIT_FOLDER = 'commits/'
+from os import listdir
+import re
+import pandas as pd
+from os.path import isfile, join, isdir
+import pickle as p
+from subprocess import Popen, PIPE
+from subprocess import CalledProcessError
+from subprocess import TimeoutExpired
+import pickle as p
+import os
+import concurrent.futures
+
+import time
+import math
+from collections import Counter
+
+import datetime
+import subprocess
+
+
+sourceCodeColumns = ['packageName', 'className', 'methodNames', 'formalParameter',
+                       'methodInvocation', 'memberReference', 'documentation', 'literal', 'rawSource', 'hunks',
+                       'commitLogs', 'classNameExt']
+
+
+def nap():
+    time.sleep(1)
+
+def setLogg():
+    # logging.basicConfig(filename='app.log', filemode='w',level=logging.DEBUG)
+    root = logging.getLogger()
+    root.setLevel(logging.DEBUG)
+
+    ch = logging.StreamHandler(sys.stdout)
+    ch.setLevel(logging.WARNING)
+    formatter = logging.Formatter('%(asctime)s - %(process)d - %(levelname)s - %(filename)s:%(funcName)s - %(message)s')
+    ch.setFormatter(formatter)
+    # ch.addFilter(lambda record: record.levelno <= logging.)
+    root.addHandler(ch)
+
+    # # 2 handlers for the same logger:
+    # h1 = logging.StreamHandler(sys.stdout)
+    # h1.setLevel(logging.DEBUG)
+    # # filter out everything that is above INFO level (WARN, ERROR, ...)
+    # h1.addFilter(lambda record: record.levelno <= logging.INFO)
+    # h1.setFormatter(formatter)
+    # root.addHandler(h1)
+
+    h2 = logging.FileHandler(filename='app.log', mode='w')
+    # take only warnings and error logs
+    h2.setLevel(logging.DEBUG)
+    h2.setFormatter(formatter)
+    root.addHandler(h2)
+
+def setEnv(args):
+    # env = args.env
+
+    # logging.info('Environment: %s',env)
+
+    os.environ["ROOT_DIR"] = args.root
+    sys.path.append(args.root)
+
+
+    import yaml
+    if os.uname().nodename != '':
+        with open(join(os.environ["ROOT_DIR"], os.uname().nodename + ".config.yml"), 'r') as ymlfile:
+            cfg = yaml.load(ymlfile)
+    else:
+        with open(join(os.environ["ROOT_DIR"], "config.yml"), 'r') as ymlfile:
+            cfg = yaml.load(ymlfile)
+
+    # for section in cfg:
+    #     print(section)
+    # print(cfg['mysql'])
+    # print(cfg['other'])
+
+    # os.environ["JDK7"] = cfg['java']['7home']
+    os.environ["JDK8"] = cfg['java']['8home']
+    os.environ["spinfer"] = cfg['spinfer']['home']
+    os.environ["coccinelle"] = cfg['coccinelle']['home']
+    os.environ["dataset"] = cfg['dataset']['home']
+    os.environ["REPO_PATH"] = cfg['dataset']['repo']
+    os.environ["DATA_PATH"] = cfg['fixminer']['datapath']
+    os.environ["PROJECT_TYPE"] = cfg['fixminer']['projectType']
+
+    # import yaml
+    #
+    # with open(join(os.environ["ROOT_DIR"],"config.yml"), 'r') as ymlfile:
+    #     cfg = yaml.load(ymlfile)
+    #
+    # # for section in cfg:
+    # #     print(section)
+    # # print(cfg['mysql'])
+    # # print(cfg['other'])
+    #
+    # os.environ["JDK7"] = cfg['java']['7home']
+    # os.environ["JDK8"] = cfg['java']['8home']
+    # os.environ["D4JHOME"] = cfg['defects4j']['home']
+
+
+    os.environ["CODE_PATH"] = join(os.environ["ROOT_DIR"],'code/')
+    # os.environ["DATA_PATH"] = join(os.environ["ROOT_DIR"],'data/')
+    # os.environ["REPO_PATH"] = join(os.environ["DATA_PATH"], 'gitrepo/')
+    os.environ["COMMIT_DFS"]= join(os.environ["DATA_PATH"],'commitsDF/')
+    os.environ["SIMI_DIR"]= join(os.environ["DATA_PATH"],'simi/')
+    os.environ["DTM_PATH"] = join(os.environ["DATA_PATH"], 'dtm/')
+    os.environ["SIMI_SINGLE"] = join(os.environ["DATA_PATH"], 'simiSingle/')
+    os.environ["FEATURE_DIR"] = join(os.environ["DATA_PATH"],'features/')
+    
+    os.environ["BUG_POINT"] = join(os.environ["DATA_PATH"], 'bugPoints/')
+    os.environ["DEFECTS4J"] = join(os.environ["DATA_PATH"], 'defects4jdata/')
+
+    os.environ["BUG_REPORT"] = join(os.environ["DATA_PATH"], 'bugReports/')
+    os.environ["BUG_REPORT_FEATURES"] = join(os.environ["DATA_PATH"], 'bugReportFeatures/')
+    # os.environ["PARSED_DIR"] = join(os.environ["CODE_PATH"], 'parsedFilesSingle/')
+    # os.environ["PARSED_M_DIR"] = join(os.environ["CODE_PATH"], 'parsedFilesMulti/')
+
+    os.environ["PARSED"] = join(os.environ["DATA_PATH"], 'parsedPj/')
+    os.environ["PARSED_DIR"] = join(os.environ["DATA_PATH"], 'parsedFilesSingle/')
+    os.environ["COMMIT_FOLDER"] = join(os.environ["DATA_PATH"], 'commits/')
+    os.environ["CLASSIFIER_DIR"] = join(os.environ["DATA_PATH"], 'classifiers/')
+    os.environ["PREDICTION_DIR"] = join(os.environ["DATA_PATH"], 'predictions/')
+    os.environ["DATASET_DIR"] = join(os.environ["DATA_PATH"], 'datasets/')
+    os.environ["REMOTE_PATH"] = '/Volumes/Samsung_T5/data'
+
+
+
+
+
+    logging.info('ROOT_DIR : %s', os.environ["ROOT_DIR"])
+    logging.info('REPO_PATH : %s', os.environ["REPO_PATH"])
+    logging.info('CODE_PATH : %s', os.environ["CODE_PATH"])
+    logging.info('COMMIT_DFS : %s', os.environ["COMMIT_DFS"])
+    # logging.info('SIMI_DIR : %s', os.environ["SIMI_DIR"])
+    logging.info('BUG_POINT : %s', os.environ["BUG_POINT"])
+    # logging.info('PARSED_DIR : %s', os.environ["PARSED_DIR"])
+    logging.info('COMMIT_FOLDER : %s', os.environ["COMMIT_FOLDER"])
+    # logging.info('DTM_PATH : %s', os.environ["DTM_PATH"])
+    # logging.info('SIMI_SINGLE : %s', os.environ["SIMI_SINGLE"])
+    logging.info('FEATURE_DIR : %s', os.environ["FEATURE_DIR"])
+    logging.info('CLASSIFIER_DIR : %s', os.environ["CLASSIFIER_DIR"])
+    logging.info('PREDICTION_DIR : %s', os.environ["PREDICTION_DIR"])
+    logging.info('DATASET_DIR : %s', os.environ["DATASET_DIR"])
+
+
+
+def getRun():
+    import argparse
+    parser = argparse.ArgumentParser(description='')
+    # parser.add_argument('-subject', dest='subject', help='Environment')
+    parser.add_argument('-root', dest='root', help='root folder')
+    parser.add_argument('-job',dest='job',help='job name')
+
+
+    args = parser.parse_args()
+
+    if args.root is None or args.job is None:
+        parser.print_help()
+        raise AttributeError
+    return args
+
+
+
+
+
+def shellCallTemplate(cmd,enc='utf-8'):
+    try:
+        logging.info(cmd)
+        with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True,encoding=enc) as p:
+            output, errors = p.communicate()
+            # print(output)
+            if errors:
+                m = re.search('unknown revision or path not in the working tree', errors)
+                if not m:
+                    raise CalledProcessError(errors, '-1')
+            output
+    except CalledProcessError as e:
+        logging.error(errors)
+    except Exception as e:
+        logging.error(e)
+    return output
+
+def getChildMem(pid,children):
+
+    out = subprocess.Popen(['pgrep', '-P', str(pid)],
+                           stdout=subprocess.PIPE).communicate()[0].split(b'\n')
+    child = out[0].decode()
+    if child !='':
+        children.append(child)
+        getChildMem(child,children)
+    else:
+        return children
+
+def getAllChildMe(pid):
+
+    childrenProcess = []
+    getChildMem(pid,childrenProcess)
+
+    # if child == '':
+    return sum(map(memory_usage_ps,childrenProcess)) + memory_usage_ps(pid)
+    # else:
+    #     return memory_usage_ps(child) + memory_usage_ps(pid)
+def memory_usage_ps(pid):
+    import subprocess
+    out = subprocess.Popen(['ps', 'v', '-p', str(pid)],
+    stdout=subprocess.PIPE).communicate()[0].split(b'\n')
+    vsz_index = out[0].split().index(b'RSS')
+    if out[1].decode() != '':
+        mem = float(out[1].split()[vsz_index]) / 1024
+    else:
+        mem = float(0)
+    return mem
+
+def raiseTime(cmd,timeout,my_timer):
+    my_timer.cancel()
+    raise TimeoutExpired(cmd, timeout)
+
+def killP(pid):
+    out = subprocess.Popen(['kill', str(pid)],
+    stdout=subprocess.PIPE).communicate()[0].split(b'\n')
+    out
+
+
+def shellGitCheckout(cmd,timeout =600,enc='utf-8'):
+    output = ''
+    errors = ''
+    # logging.debug(cmd)
+
+    with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True,encoding=enc) as p:
+        try:
+            output, errors = p.communicate(timeout=timeout)
+            # print(output)
+            logging.debug(cmd + '\t' +output)
+            # logging.info(errors)
+            if errors:
+                raise CalledProcessError(errors, '-1')
+            output
+        except CalledProcessError as e:
+            logging.debug(cmd +'\t'+ errors)
+        except TimeoutExpired as t:
+            p.terminate()
+            p.communicate()
+            # p.kill()
+            logging.warning(cmd +'\t'+str(t))
+    return output,errors
+
+def callSpinfer(cmd,timeout =600,enc='utf-8'):
+    output = ''
+    errors = ''
+    # logging.debug(cmd)
+    my_timer = None
+    with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True,encoding=enc) as p:
+        try:
+            start = datetime.datetime.now()
+            memusage = getAllChildMe(p.pid)
+            # isExit = False
+            while(memusage != 0.0):
+                end = datetime.datetime.now()
+                elapsed = end - start
+                if(elapsed.seconds > timeout):
+                    raise TimeoutExpired(cmd,timeout)
+                memusage = getAllChildMe(p.pid)
+                # print(str(p.pid) + " ; " + str(memusage))
+                if memusage > 2000:
+                    # isExit = True
+                    raise TimeoutExpired(cmd,timeout)
+
+            output, errors = p.communicate(timeout=timeout)
+            # print(output)
+            logging.debug(cmd + '\t' +output)
+            # logging.info(errors)
+            if errors:
+                raise CalledProcessError(errors, '-1')
+            output
+        except CalledProcessError as e:
+            logging.debug(cmd +'\t'+ errors)
+        except TimeoutExpired as t:
+            # my_timer.cancel()
+
+
+            childrenProcess = []
+            getChildMem(p.pid, childrenProcess)
+            [killP(i) for i in childrenProcess]
+
+            p.terminate()
+            p.communicate()
+            # p.kill()
+            logging.warning(cmd +'\t'+str(t))
+    return output,errors
+
+def save_zipped_pickle(obj, filename, protocol=-1):
+    with gzip.open(filename, 'wb') as f:
+        p.dump(obj, f, protocol)
+
+def load_zipped_pickle(filename):
+    with gzip.open(filename, 'rb') as f:
+        loaded_object = p.load(f)
+        return loaded_object
+
+def file2path(file):
+    count = file.count(".") - 1
+    file = file.replace('.', '/', count)
+    return file
+
+def isFileInList(file,checkList):
+    for f in checkList:
+        if f in file:
+            return True
+    return False
+    # [i for i in ansFiles if 'org/fusesource/esb/itests/basic/fabric/EsbFeatureTest.java' in i]
+
+def get_venn_sections(sets):
+    """
+    Given a list of sets, return a new list of sets with all the possible
+    mutually exclusive overlapping combinations of those sets.  Another way
+    to think of this is the mutually exclusive sections of a venn diagram
+    of the sets.  If the original list has N sets, the returned list will
+    have (2**N)-1 sets.
+
+    Parameters
+    ----------
+    sets : list of set
+
+    Returns
+    -------
+    combinations : list of tuple
+        tag : str
+            Binary string representing which sets are included / excluded in
+            the combination.
+        set : set
+            The set formed by the overlapping input sets.
+    """
+    num_combinations = 2 ** len(sets)
+    bit_flags = [2 ** n for n in range(len(sets))]
+    flags_zip_sets = [z for z in zip(bit_flags, sets)]
+
+    #combo_sets = []
+    combo_sets = dict()
+    for bits in range(num_combinations - 1, 0, -1):
+        include_sets = [s for flag, s in flags_zip_sets if bits & flag]
+        exclude_sets = [s for flag, s in flags_zip_sets if not bits & flag]
+        combo = set.intersection(*include_sets)
+        combo = set.difference(combo, *exclude_sets)
+        tag = ''.join([str(int((bits & flag) > 0)) for flag in bit_flags])
+        #combo_sets.append((tag, combo))
+        combo_sets[tag] = combo
+    return combo_sets
+
+def pairwise(iterable):
+    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
+    a, b = itertools.tee(iterable)
+    next(b, None)
+    return zip(a, b)
+
+def RR_XGB(x,ao,column):
+    if x[ao] == 1:
+        return (1.0 / (x[column]))
+    elif pd.isnull(x[ao]):
+        return None
+    else:
+        return 0
+
+def parallelRunNo(coreFun,elements,*args):
+    with concurrent.futures.ProcessPoolExecutor(max_workers=int(8)) as executor:
+        try:
+            futures = {executor.submit(coreFun, l,*args): l for l in elements}
+
+            kwargs = {
+                'total': len(futures),
+                'unit': 'files',
+                'unit_scale': True,
+                'leave': False
+            }
+
+            for future in tqdm(concurrent.futures.as_completed(futures), **kwargs):
+                url = futures[future]
+                try:
+                    data = future.result(timeout=None)
+
+
+                except Exception as exc:
+                    logging.error('%r generated an exception: %s' % (url, exc))
+                    raise
+
+        except Exception as e:
+            logging.error(e)
+            executor.shutdown()
+            raise
+
+
+def parallelRun(coreFun,elements,*args,max_workers=os.cpu_count()):
+    with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
+        try:
+            futures = {executor.submit(coreFun, l,*args): l for l in elements}
+
+            kwargs = {
+                'total': len(futures),
+                'unit': 'files',
+                'unit_scale': True,
+                'leave': False
+            }
+
+            for future in tqdm(concurrent.futures.as_completed(futures), **kwargs):
+                url = futures[future]
+                try:
+                    data = future.result()
+                except Exception as exc:
+                    logging.error('%r generated an exception: %s' % (url, exc))
+                    raise
+        except Exception as e:
+            logging.error(e)
+            executor.shutdown()
+            raise
+
+
+def parallelRunMerge(coreFun,elements,*args,max_workers=os.cpu_count()):
+    dataL = []
+    with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
+        try:
+            futures = {executor.submit(coreFun, l,*args): l for l in elements}
+            kwargs = {
+                'total': len(futures),
+                'unit': 'files',
+                'unit_scale': True,
+                'leave': False
+            }
+            for future in tqdm(concurrent.futures.as_completed(futures), **kwargs):
+                url = futures[future]
+                try:
+                    data = future.result()
+                    dataL.append(data)
+
+                except Exception as exc:
+                    logging.error('%r generated an exception: %s' % (url, exc))
+                    raise
+
+            return dataL
+        except Exception as e:
+            logging.error(e)
+            executor.shutdown()
+            raise
+
+
+def parallelRunMergeNew(coreFun,elements,*args,max_workers=os.cpu_count()):
+
+    res = []
+    with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
+        try:
+            futures = {executor.submit(coreFun, l,*args): l for l in elements}
+
+            kwargs = {
+                'total': len(futures),
+                'unit': 'files',
+                'unit_scale': True,
+                'leave': False
+            }
+
+            for future in tqdm(concurrent.futures.as_completed(futures), **kwargs):
+                url = futures[future]
+                try:
+                    data = future.result()
+                    res.append(data)
+                except Exception as exc:
+                    logging.error('%r generated an exception: %s' % (url, exc))
+                    raise
+
+        except Exception as e:
+            logging.error(e)
+            executor.shutdown()
+            raise
+    aDF = pd.concat(res)
+    return aDF
+
+def get_filepaths(directory,extension):
+
+    file_paths = []  # List which will store all of the full filepaths.\n,
+    exclude = '.git'
+    # Walk the tree.\n,
+    for root, directories, files in os.walk(directory):
+        directories[:] = [d for d in directories if d not in exclude]
+        # java = [i for i in files if i.endswith(extension)]
+        java = [i for i in files if bool(re.search(extension, i))]
+
+        for filename in java:
+            # Join the two strings in order to form the full filepath.\n,
+            filepath = os.path.join(root, filename)
+            file_paths.append(filepath)  # Add it to the list.\n,
+
+    return file_paths  # Self-explanatory.\n,
+
+def get_class_weights(y):
+    counter = Counter(y)
+    majority = max(counter.values())
+    return  {cls: round(float(majority)/float(count), 2) for cls, count in counter.items()}
+
+
+def stopDB(dbDir,portInner,dbName):
+    cmd = "bash " + dbDir + "/" + "stopServer.sh " + " " + portInner;
+
+    o, e = shellGitCheckout(cmd)
+    logging.info(o)
+
+
+
+def startDB(dbDir,portInner,projectType):
+    dbName = "dumps-"+projectType+".rdb"
+    # portInner = '6380'
+    cmd = "bash " + dbDir + "/" + "startServer.sh " + dbDir + " "+dbName+ " " + portInner;
+
+    o, e = shellGitCheckout(cmd)
+    ping = "redis-cli -p "+portInner+" ping"
+    o, e = shellGitCheckout(ping)
+    m = re.search('PONG', o)
+
+    while not m:
+        time.sleep(10)
+        logging.info('Waiting for checkout')
+        o, e = shellGitCheckout(ping)
+        m = re.search('PONG', o)
+    print(o)
+
+
+def unique_everseen(iterable, key=None):
+    "List unique elements, preserving order. Remember all elements ever seen."
+    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    seen = set()
+    seen_add = seen.add
+    if key is None:
+        for element in itertools.filterfalse(seen.__contains__, iterable):
+            seen_add(element)
+            yield element
+    else:
+        for element in iterable:
+            k = key(element)
+            if k not in seen:
+                seen_add(k)
+                yield element
+
+def plotBox(yList,labels, fn, rotate=False,limit=True):
+    import matplotlib
+    matplotlib.use("TkAgg")
+    import matplotlib.pyplot as plt
+
+
+
+    fig = plt.figure()
+    ax1 = fig.add_subplot(111)
+    meanpointsprops = dict(markeredgecolor ='blue',markerfacecolor=
+                           'blue')
+
+    flierprops = dict(markeredgecolor ='black',markerfacecolor=
+                           'black',marker='.',markersize=2)
+    box = ax1.boxplot(yList, 0, flierprops=flierprops,widths=0.5, showmeans=False, vert=True,meanprops=meanpointsprops)
+    for line in box['medians']:
+        x,y = line.get_xydata()[1]
+        line.set(linewidth=3)
+        line.set_color('blue')
+    # plt.scatter(labels, yList, color='r')
+    # plt.plot(labels, yList, '-o')
+    # ax1.boxplot(yList,notch=False, sym='', vert=True, whis=1.5,
+    #     positions=None, widths=None, patch_artist=True,
+    #     bootstrap=None, usermedians=None, conf_intervals=None)
+    if rotate:
+        ax1.set_xticklabels(labels, rotation=45, ha='right')
+    else:
+        # ax1.set_xticklabels(labels)
+        # ax1.set_xticklabels(None)
+        ax1.get_xaxis().set_ticklabels([])
+    # sns.boxplot(yList, ax=ax1)
+    if limit:
+        ax1.set_ylim(top=1.1,bottom=0)
+        ax1.yaxis.set_ticks([0.0,1.0])
+    else:
+        ax1.set_yscale('log')
+        ax1.set_xlabel('Cluster Member Size')
+        ax1.set_ylabel('Folds')
+    plt.ion()
+
+    plt.subplots_adjust(wspace=0, hspace=0)
+    fig = plt.gcf()
+
+    # fig.tight_layout()
+    fig.set_size_inches(7, 1, forward=True)
+    fig.savefig(fn, dpi=100, bbox_inches='tight')
+
+
+    plt.show()
+
+def plotBox2(ys,labels, fn,means, rotate=False,limit=True):
+
+
+    import matplotlib
+    matplotlib.use("TkAgg")
+    import matplotlib.pyplot as plt
+
+
+    fig,axes = plt.subplots(nrows=3,ncols=1)
+
+    for ax1,yList,l,l2,mean in zip(axes.flat,ys,labels,['Shapes','Actions','Tokens'],means):
+        # plt.setp(ax1.get_xticks(),visible=False)
+        # ax1 = fig.add_subplot(111)
+        meanpointsprops = dict(markeredgecolor ='blue',markerfacecolor=
+                               'blue')
+
+        flierprops = dict(markeredgecolor ='black',markerfacecolor=
+                               'black',marker='.',markersize=2)
+        box = ax1.boxplot(yList, 0, flierprops=flierprops,widths=0.5, showmeans=False, vert=True,meanprops=meanpointsprops)
+
+        ax1.axhline(linewidth=2, color='r',y=mean)
+
+        for line in box['medians']:
+            x,y = line.get_xydata()[1]
+            line.set(linewidth=3)
+            line.set_color('blue')
+        # plt.scatter(labels, yList, color='r')
+        # plt.plot(labels, yList, '-o')
+        # ax1.boxplot(yList,notch=False, sym='', vert=True, whis=1.5,
+        #     positions=None, widths=None, patch_artist=True,
+        #     bootstrap=None, usermedians=None, conf_intervals=None)
+        if rotate:
+            ax1.set_xticklabels(l, rotation=45, ha='right')
+        else:
+            # ax1.set_xticklabels(labels)
+            # ax1.set_xticklabels(None)
+            ax1.get_xaxis().set_ticklabels([])
+            # ax1.get_xaxis().set_ticks([])
+        # sns.boxplot(yList, ax=ax1)
+        if limit:
+            if l2 !='Tokens':
+                ax1.set_ylim(top=1,bottom=0)
+            else:
+                ax1.set_ylim(top=1.1, bottom=0)
+            ax1.yaxis.set_ticks([0.0,mean,0.5,1.0])
+            ax1.yaxis.set_ticklabels([0,'',0.5,1])
+            ax1.tick_params(direction='out', length=6, width=2, axis='y',
+                         grid_color='r', grid_alpha=0.5)
+
+        else:
+            # ax1.set_yscale('log')
+            ax1.set_xlabel('Cluster Member Size')
+            ax1.set_ylabel('Folds')
+        ax1.set_aspect('auto')
+
+        ax1.set_ylabel(l2)
+    labels = ['C-'+str(i+1) for i in labels[0]]
+    ax1.set_xticklabels(labels)
+    ax1.set_xticklabels(labels, rotation=45, ha='right')
+    # plt.setp(ax1.get_xticks(), visible=True)
+
+    ax1.set_xlabel('Clusters')
+    # ax1.set_xlabel('Similarity')
+    plt.ion()
+
+    plt.subplots_adjust(wspace=0, hspace=0.05)
+    fig = plt.gcf()
+
+
+    # fig.tight_layout()
+    fig.set_size_inches(7, 7, forward=True)
+    fig.savefig(fn, dpi=100, bbox_inches='tight')
+
+
+    plt.show()
+
+
+def plotScatter(s1,s2,vs,label,limits,type):
+    import matplotlib
+    matplotlib.use("TkAgg")
+    import matplotlib.pyplot as plt
+    plt.figure(figsize=(2.5, 2.5))
+
+    # s1 = (df1[whichColumn]).apply(lambda x: round(Decimal(x), 3))
+    # s2 = (df2[whichColumn]).apply(lambda x: round(Decimal(x), 3))
+    plt.scatter(s1, s2, color='r')
+    # plt.text(0.5, 0.25, '\\textbf{%s wins}' %
+    #          vs, ha='center', va='center', rotation=45)
+    # plt.text(0.2, 0.5, '\\textbf{%s wins}' %
+    #          label, ha='center', va='center', rotation=45)
+    # plt.title('%s vs. %s ' % (label, vs))
+    ax = plt.gca()
+    ax.grid(False)
+    # ax.patch.set_alpha(0)
+    ax.set_xlim([0, limits])
+    ax.set_ylim([0, limits])
+    start, end = ax.get_xlim()
+    stepsize = 1
+    ax.xaxis.set_ticks(np.arange(0, end, stepsize))
+    ax.yaxis.set_ticks(np.arange(0, end, stepsize))
+    x = np.linspace(start, end, limits+1)
+    y = np.linspace(start, end, limits+1)
+    ax.fill_between(x, y, end, facecolor='b', alpha=0.3)
+    # plt.plot(np.linspace(0, 1, 10), np.linspace(0, 1, 10), lw=1)
+    ax.spines['top'].set_visible(True)
+    ax.spines['right'].set_visible(True)
+    ax.spines['bottom'].set_visible(True)
+    ax.spines['left'].set_visible(True)
+
+    ax.set_xlabel(vs)
+    ax.set_ylabel(label)
+
+    plt.subplots_adjust(wspace=0, hspace=0)
+    # fig = plt.gcf()
+    # fig.set_size_inches(4, 4, forward=True)
+
+    plt.savefig(
+        os.path.join(
+            'scatter',
+            '{}.pdf'.format(
+                type).replace(' ', '-')
+        ),
+        tight_bbox=True
+    )
+
+import threading
+class BackgroundTask(object):
+    """ Threading example class
+    The run() method will be started and it will run in the background
+    until the application exits.
+    """
+
+    def __init__(self, model,PATH, interval=1):
+        """ Constructor
+        :type interval: int
+        :param interval: Check interval, in seconds
+        """
+        self.interval = interval
+        self.model = model
+        self.path = PATH
+
+        thread = threading.Thread(target=self.run, args=())
+        thread.daemon = True                            # Daemonize thread
+        thread.start()                                  # Start the execution
+
+    def run(self):
+        """ Method that runs forever """
+        self.model.save_model(self.path,
+                     num_iteration=self.model.best_iteration)

python/common/preprocessing.py

@@ -0,0 +1,300 @@
+from nltk.tokenize import RegexpTokenizer
+# from stop_words import get_stop_words
+from nltk.stem.porter import PorterStemmer
+from string import punctuation
+import re
+from nltk.corpus import stopwords
+en_stop = stopwords.words('english')
+from nltk.corpus import wordnet
+import html
+
+from common.commons import *
+CODE_PATH = os.environ["CODE_PATH"]
+
+# import spacy
+# nlp = spacy.load('en_core_web_lg', disable=['parser', 'tagger', 'ner'])
+# nlp.max_length =100000000
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+import sys
+
+
+def getTokensForPatterns(res):
+    if isinstance(res, list):
+        merged = str()
+        for r in res:
+            if isinstance(r, list):
+                merged = merged + ' ' + ' '.join(r)
+            else:
+                merged = merged +' ' + r
+    else:
+        merged=res
+
+    res = html.unescape(merged)
+
+    tokens = getTokens(res,False)
+
+    stripped = []
+    for t in tokens:
+        splits = re.split('\.|\(|\)|:|>|<|:|=|/|\\\\|\'|-|,|\]|\[|}|{|;',t)
+        for s in splits:
+            stripped.append(s)
+    non_empty = [i for i in stripped if i != '']
+    return non_empty
+
+def preprocessingCodeElementsList(res):
+    printDetail = False
+    if isinstance(res, list):
+        merged = str()
+        for r in res:
+            if isinstance(r, list):
+                merged = merged + ' ' + ' '.join(r)
+            else:
+                merged = merged +' ' + r
+    else:
+        merged=res
+
+    res = html.unescape(merged)
+
+    tokens = getTokens(res,printDetail)
+
+    stripped = []
+    for t in tokens:
+        splits = re.split('\.|\(|\)|:|>|<|:|=|/|\\\\|\'|-|,|\]|\[',t)
+        for s in splits:
+            stripped.append(s)
+    punc = removeEndingPunct(stripped,printDetail)
+
+    non_empty = [i for i in punc if i != '']
+
+    stripped = removeEndingPunct(non_empty,printDetail)
+
+    camelCase = handleCamelCase(stripped,printDetail,True)
+
+    underScore = handleUnderScore(camelCase,printDetail,True)
+
+    lower = [i.lower() for i in underScore]
+
+    # stopped_tokens = [i for i in lower if not i in en_stop]
+
+    stem2 = stem(lower,printDetail)
+    if printDetail:
+        print('=====CLEANED=========')
+        print(stem2)
+
+    return stem2
+
+def preprocessingForSimi(res):
+    printDetail = False
+    if isinstance(res, list):
+        merged = str()
+        for r in res:
+            if isinstance(r, list):
+                merged = merged + ' ' + ' '.join(r)
+            else:
+                merged = merged +' ' + r
+    else:
+        merged=res
+
+    res = html.unescape(merged)
+
+    tokens = getTokens(res,printDetail)
+
+    stripped = []
+    for t in tokens:
+        splits = re.split('\.|\(|\)|:|>|<|:|=|/|\\\\|\'|-|,|\]|\[',t)
+        for s in splits:
+            stripped.append(s)
+    punc = removeEndingPunct(stripped,printDetail)
+
+    non_empty = [i for i in punc if i != '']
+
+    stripped = removeEndingPunct(non_empty,printDetail)
+
+    camelCase = handleCamelCase(stripped,printDetail,False)
+
+    underScore = handleUnderScore(camelCase,printDetail,False)
+
+    # lower = [i.lower() for i in underScore]
+
+    # stopped_tokens = [i for i in lower if not i in en_stop]
+
+    # stem2 = stem(lower,printDetail)
+    # if printDetail:
+    #     print('=====CLEANED=========')
+    #     print(stem2)
+
+    return underScore
+
+def preprocessingNL(res):
+    try:
+        printDetail = False
+
+        if isinstance(res, list):
+            merged = str()
+            for r in res:
+                if isinstance(r, list):
+                    merged = merged + ' ' + ' '.join(r)
+                else:
+                    merged = merged +' ' + r
+        else:
+            merged=res
+
+        res = html.unescape(merged)
+        html_decoded_string = res.replace("&amp;", "&").replace("&quot;", '"').replace("&apos;", "'").replace("&gt;",
+                                                                                                               ">").replace(
+            "&lt;", "<")
+        html_decoded_string = re.sub(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', '',html_decoded_string)
+
+        tokens = getTokens(html_decoded_string,printDetail)
+
+        stripped = []
+        for t in tokens:
+            splits = re.split('\.|\(|\)|:|>|<|:|=|/|\\\\|\'|-|,|\]|\[',t)
+            # splits = re.split('\.|\(|\)|:|>|<|:|=|/|\\\\|\'|-',t)
+            for s in splits:
+                stripped.append(s)
+        punc = removeEndingPunct(stripped,printDetail)
+
+        non_empty = [i for i in punc if i != '']
+
+        stripped = removeEndingPunct(non_empty,printDetail)
+
+        camelCase = handleCamelCase(stripped,printDetail,True)
+
+        underScore = handleUnderScore(camelCase,printDetail,True)
+
+        lower = [i.lower() for i in underScore]
+
+        stopped_tokens = [i for i in lower if not i in en_stop]
+
+        nonDigit = [i for i in stopped_tokens if (not i.isdigit())]
+
+        # doc = nlp(' '.join(nonDigit))
+        # newWord = []
+        # for token in doc:
+        #     if(token.text in nlp.vocab):
+        #         newWord.append(token.text)
+
+        stem2 = stem(nonDigit,printDetail)
+
+        if printDetail:
+            print('=====CLEANED=========')
+            print(stem2)
+
+        return stem2
+    except Exception as e:
+        logging.error(e)
+
+def getTokens(re,printDetail=False):
+    tokenizer = RegexpTokenizer(r'\S+')
+    tokens = tokenizer.tokenize(re)
+    if printDetail:
+        print('=====TOKENS=========')
+        print(tokens)
+
+    return tokens
+
+def charLength(x, l=3):
+    if x.isalpha() and len(x) >= l:
+        return True
+    else:
+        return False
+
+
+def removeEndingPunct(re,printDetail):
+    stripped = [i.strip(punctuation) for i in re]
+    if printDetail:
+        print('=====removeEndingPunct=========')
+        print(stripped)
+    return stripped
+
+def handleCamelCase(re,printDetail=False,keepOriginal = False):
+    camelCased = list()
+
+    for i in re:
+        listOfCC = camel_case_split(i)
+        camelCased.extend(listOfCC)
+        if i not in listOfCC and keepOriginal:
+            camelCased.append(i)
+
+    if printDetail:
+        print('=====CAMEL CASE=========')
+        print(camelCased)
+    return camelCased
+
+def handleUnderScore(re,printDetail=False,keepOriginal = False):
+    underScored = list()
+    for i in re:
+        listOfCC = i.split('_')
+        underScored.extend(listOfCC)
+        if i not in listOfCC and keepOriginal:
+            underScored.append(i)
+
+    if printDetail:
+        print('=====UNDER SCORE=========')
+        print(underScored)
+
+    return underScored
+
+def camel_case_split(identifier):
+    matches = re.finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)', identifier)
+    res = [m.group(0) for m in matches]
+
+    return res
+
+def stem(res,printDetail):
+    p_stemmer = PorterStemmer()
+    stemmed_tokens = [p_stemmer.stem(i.strip()) for i in res if i]
+    if printDetail:
+        print('=====STEMMED=========')
+        print(stemmed_tokens)
+    return stemmed_tokens
+
+def isEnglish(word_to_test):
+    if not wordnet.synsets(word_to_test):
+        #Not an English Word
+        #TODO
+        word_to_test
+        #print word_to_test
+    else:
+        return word_to_test
+
+
+def dummy_fun(doc):
+    return doc
+
+def calculateTfIdfCodeElementsList(aCorpus):
+    global progress
+    progress = 0
+    v = TfidfVectorizer(tokenizer=dummy_fun,stop_words=None,lowercase=False,sublinear_tf=True)#,max_df=0.7,min_df=3)
+    m = v.fit(aCorpus)
+    return v
+
+def calculateTfIdfNLList(aCorpus):
+    global progress
+    progress = 0
+    v = TfidfVectorizer(tokenizer=dummy_fun,stop_words=None,lowercase=False,sublinear_tf=True)#,max_df=0.7,min_df=3)
+    m = v.fit(aCorpus)
+    return v
+
+def getDTMNL(x,v,corpus):
+    ind =x.name
+    v.tokenizer = dummy_fun
+    return v.transform([corpus[ind]])
+def getDTMCE(x,v,corpus):
+    ind =x.name
+    v.tokenizer = dummy_fun
+    return v.transform([corpus[ind]])
+
+def getBRDTM(x,v,corpus):
+    ind =x.name
+    v.tokenizer = dummy_fun
+    return v.transform([corpus[ind]])
+
+
+def getBRDTMCEs(x,v,corpus):
+    ind =x.name
+    v.tokenizer = dummy_fun
+    return v.transform([corpus[ind]])

python/config.yml

@@ -0,0 +1,17 @@
+java:
+    8home:  /Library/Java/JavaVirtualMachines/jdk1.8.0_181.jdk/Contents/Home
+spinfer:
+    home:   /Users/anil.koyuncu/projects/fixminer/spinfer/spinfer.native
+    #home:   /Users/anil.koyuncu/projects/spinfer/spinfer.native
+dataset:
+#    home:   /Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/
+    home:   /data/fixminer-core/python/data/gumInputLinux/
+
+coccinelle:
+    home:   /Users/anil.koyuncu/projects/fixminer/spinfer/statics
+    #home:   /Users/anil.koyuncu/projects/spinfer/statics
+
+
+fixminer:
+#    datapath: /Users/anil.koyuncu/projects/test/fixminer-core/python/data/
+    datapath: /data/fixminer-core/python/data/

python/data/app.properties

@@ -0,0 +1,16 @@
+pjName = gumInput
+portInner = 6380
+portDumps = 6399
+parallelism = AKKA
+numOfWorkers = 20
+hostname = localhost
+hunkLimit = 1
+
+#inputPath = /Users/anilkoyuncu/projects/gumInputLinux
+inputPath = /data/fixminer-core/python/data/gumInputLinux
+redisPath = /data/fixminer-core/python/data/redis
+#srcMLPath=  /Users/anil.koyuncu/Downloads/srcML/src2srcml
+#srcMLPath=  /Users/anil.koyuncu/Downloads/srcML.0.9.5/bin/srcml
+srcMLPath=  /usr/local/bin/srcml
+actionType = ALL
+eDiffTimeout = 900

python/data/buildBugzoo.sh

@@ -0,0 +1,5 @@
+#!/bin/bash 
+source activate fixminerEnv
+
+bugzoo bug build $1
+

python/data/classification.json

@@ -0,0 +1,371 @@
+{
+	"Repair Actions": {
+		"Assignment": {
+			"assignAdd": {
+				"name": "Addition",
+				"fullname": "Assignment addition"
+			},
+			"assignRem": {
+				"name": "Removal",
+				"fullname": "Assignment removal"
+			},
+			"assignExpChange": {
+				"name": "Expression modification",
+				"fullname": "Assignment expression modification"
+			}
+		},
+		"Conditional": {
+			"condBranIfAdd": {
+				"name": "if branch addition",
+				"fullname": "Conditional (if) branch addition"
+			},
+			"condBranIfElseAdd": {
+				"name": "if-else branches addition",
+				"fullname": "Conditional (if-else) branches addition"
+			},
+			"condBranElseAdd": {
+				"name": "else branch addition",
+				"fullname": "Conditional (else) branch addition"	
+			},
+			"condBranCaseAdd": {
+				"name": "case branch addition",
+				"fullname": "Conditional (case in switch) branch addition"	
+			},
+			"condBranRem": {
+				"name": "Removal",
+				"fullname": "Conditional (if or else) branch removal"		
+			},
+			"condExpExpand": {
+				"name": "Expression expansion",
+				"fullname": "Conditional expression expansion"
+			},
+			"condExpRed": {
+				"name": "Expression reduction",
+				"fullname": "Conditional expression reduction"
+			},
+			"condExpMod": {
+				"name": "Expression modification",
+				"fullname": "Conditional expression modification"
+			}
+		},
+		"Loop": {
+			"loopAdd": {
+				"name": "Addition",
+				"fullname": "Loop addition"
+			},
+			"loopRem": {
+				"name": "Removal",
+				"fullname": "Loop removal"
+			},
+			"loopCondChange": {
+				"name": "Conditional expression modification",
+				"fullname": "Loop conditional expression modification"
+			},
+			"loopInitChange": {
+				"name": "Initialization field modification",
+				"fullname": "Loop initialization field modification"
+			}
+		},
+		"Method Call": {
+			"mcAdd": {
+				"name": "Addition",
+				"fullname": "Method call addition"
+			},
+			"mcRem": {
+				"name": "Removal",
+				"fullname": "Method call removal"
+			},
+			"mcRepl": {
+				"name": "Replacement",
+				"fullname": "Method call replacement"
+			},
+			"mcMove": {
+				"name": "Moving",
+				"fullname": "Method call moving"
+			},
+			"mcParAdd": {
+				"name": "Parameter addition",
+				"fullname": "Method call parameter addition"
+			},
+			"mcParRem": {
+				"name": "Parameter removal",
+				"fullname": "Method call parameter removal"
+			},
+			"mcParSwap": {
+				"name": "Parameter value swapping",
+				"fullname": "Method call parameter value swapping"
+			},
+			"mcParValChange": {
+				"name": "Parameter value modification",
+				"fullname": "Method call parameter value modification"
+			}
+		},
+		"Method Definition": {
+			"mdAdd": {
+				"name": "Addition",
+				"fullname": "Method definition addition"
+			},
+			"mdRem": {
+				"name": "Removal",
+				"fullname": "Method definition removal"
+			},
+			"mdRen": {
+				"name": "Renaming",
+				"fullname": "Method definition renaming"
+			},
+			"mdParAdd": {
+				"name": "Parameter addition",
+				"fullname": "Parameter addition in method definition"
+			},
+			"mdParRem": {
+				"name": "Parameter removal",
+				"fullname": "Parameter removal from method definition"
+			},
+			"mdParTyChange": {
+				"name": "Parameter type modification",
+				"fullname": "Parameter type modification in method definition"
+			},
+			"mdRetTyChange": {
+				"name": "Return type modification",
+				"fullname": "Method return type modification"
+			},
+			"mdModChange": {
+				"name": "Modifier change",
+				"fullname": "Method modifier change"
+			},
+			"mdOverride": {
+				"name": "Method overriding related change",
+				"fullname": "Method overriding addition or removal"
+			}
+		},
+		"Object Instantiation": {
+			"objInstAdd": {
+				"name": "Addition",
+				"fullname": "Object instantiation addition"
+			},
+			"objInstRem": {
+				"name": "Removal",
+				"fullname": "Object instantiation removal"
+			},
+			"objInstMod": {
+				"name": "Modification",
+				"fullname": "Object instantiation modification"
+			}
+		},
+		"Exception": {
+			"exTryCatchAdd": {
+				"name": "try-catch addition"
+			},
+			"exTryCatchRem": {
+				"name": "try-catch removal"
+			},
+			"exThrowsAdd": {
+				"name": "throw addition"
+			},
+			"exThrowsRem": {
+				"name": "throw removal"
+			}
+		},
+		"Return": {
+			"retBranchAdd": {
+				"name": "Addition",
+				"fullname": "Return statement addition"
+			},
+			"retRem": {
+				"name": "Removal",
+				"fullname": "Return statement removal"
+			},
+			"retExpChange": {
+				"name": "Expression modification",
+				"fullname": "Return expression modification"
+			}
+		},
+		"Variable": {
+			"varAdd": {
+				"name": "Addition",
+				"fullname": "Variable addition"
+			},
+			"varRem": {
+				"name": "Removal",
+				"fullname": "Variable removal"
+			},
+			"varTyChange": {
+				"name": "Type change",
+				"fullname": "Variable type change"
+			},
+			"varModChange": {
+				"name": "Modifier change",
+				"fullname": "Variable modifier change"
+			},
+			"varReplVar": {
+				"name": "Replacement by Another Variable",
+				"fullname": "Variable replacement by another variable"
+			},
+			"varReplMc": {
+				"name": "Replacement by Method Call",
+				"fullname": "Variable replacement by method call"
+			}
+		},
+		"Type": {
+			"tyAdd": {
+				"name": "Type addition",
+				"fullname": "Type addition"
+			},
+			"tyImpInterf": {
+				"name": "Type implemented interface modification",
+				"fullname": "Type implemented interface modification"
+			}
+		}
+	},
+	"Repair Patterns": {
+		"Conditional Block": {
+			"condBlockOthersAdd": {
+				"name": "Conditional block addition",
+				"fullname": "Conditional block addition"
+			},
+			"condBlockRetAdd": {
+				"name": "Addition with return statement",
+				"fullname": "Conditional block addition with return statement"
+
+			},
+			"condBlockExcAdd": {
+				"name": "Addition with exception throwing",
+				"fullname": "Conditional block addition with exception throwing"
+			},
+			"condBlockRem": {
+				"name": "Conditional block removal",
+				"fullname": "Conditional block removal"
+			}
+		},
+		"Expression Fix": {
+			"expLogicExpand": {
+				"name": "Logic expression expansion",
+				"fullname": "Logic expression expansion"
+			},
+			"expLogicReduce": {
+				"name": "Logic expression reduction",
+				"fullname": "Logic expression reduction"
+			},
+			"expLogicMod": {
+				"name": "Logic expression modification",
+				"fullname": "Logic expression modification"
+			},
+			"expArithMod": {
+				"name": "Arithmetic expression modification",
+				"fullname": "Arithmetic expression modification"
+			}
+		},
+		"Wraps-with/Unwraps-from": {
+			"wrapsIf": {
+				"name": "Wraps-with if statement",
+				"fullname": "Wraps-with if statement"
+			},
+			"wrapsIfElse": {
+				"name": "Wraps-with if-else statement",
+				"fullname": "Wraps-with if-else statement"
+			},
+			"wrapsElse": {
+				"name": "Wraps-with else statement",
+				"fullname": "Wraps-with else statement"
+			},
+			"wrapsTryCatch": {
+				"name": "Wraps-with try-catch block",
+				"fullname": "Wraps-with try-catch block"
+			},
+			"wrapsMethod": {
+				"name": "Wraps-with method call",
+				"fullname": "Wraps-with method call"
+			},
+			"wrapsLoop": {
+				"name": "Wraps-with loop",
+				"fullname": "Wraps-with loop"
+			},
+			"unwrapIfElse": {
+				"name": "Unwraps-from if-else statement"
+			},
+			"unwrapMethod": {
+				"name": "Unwraps-from method call"
+			},
+			"unwrapTryCatch": {
+				"name": "Unwraps-from try-catch block"
+			}
+		},
+		"Wrong Reference": {
+			"wrongVarRef": {
+				"name": "Variable",
+				"fullname": "Wrong Variable Reference"
+			},
+			"wrongMethodRef": {
+				"name": "Method",
+				"fullname": "Wrong Method Reference"
+			}
+		},
+		"Missing Null-Check": {
+			"missNullCheckP": {
+				"name": "Missing null check",
+				"fullname": "Missing null check addition"
+			},
+			"missNullCheckN": {
+				"name": "Missing non-null check",
+				"fullname": "Missing non-null check addition"
+			}
+		},
+		"Other": {
+			"singleLine": {
+				"name": "Single Line"
+			},
+			"copyPaste": {
+				"name": "Copy/Paste"
+			},
+			"constChange": {
+				"name": "Constant Change"
+			},
+			"codeMove": {
+				"name": "Code Moving"
+			},
+			"notClassified": {
+				"name": "Not classified"
+			}
+		}
+	},
+	"Runtime Information": {
+		"Automatic Repair": {
+			"rtAcs": {
+				"name": "ACS"
+			},
+			"rtCardumen": {
+				"name": "Cardumen"
+			},
+			"rtDeepRepair": {
+				"name": "DeepRepair"
+			},
+			"rtDynaMoth": {
+				"name": "DynaMoth"
+			},
+			"rtElixir": {
+				"name": "Elixir"
+			},
+			"rtGPFL": {
+				"name": "GPFL"
+			},
+			"rtHDRepair": {
+				"name": "HDRepair"
+			},
+			"rtGenProg": {
+				"name": "jGenProg"
+			},
+			"rtKali": {
+				"name": "jKali"
+			},
+			"rtNopol": {
+				"name": "Nopol"
+			},
+			"rtssFix": {
+				"name": "ssFix"
+			},
+			"rtJAID": {
+				"name": "JAID"
+			}
+		}
+	}
+}

python/data/datasets.csv

@@ -0,0 +1,29 @@
+Name,GitRepo
+libtiff, https://gitlab.com/libtiff/libtiff.git
+FFmpeg, https://github.com/FFmpeg/FFmpeg.git
+cmake, https://gitlab.kitware.com/cmake/cmake.git
+redis, https://github.com/antirez/redis.git
+ompi, https://github.com/open-mpi/ompi.git
+gzip, https://git.savannah.gnu.org/git/gzip.git
+libarchive, https://github.com/libarchive/libarchive
+cairo, https://gitlab.freedesktop.org/cairo/cairo.git
+vlc, https://github.com/videolan/vlc.git
+freebsd, https://github.com/freebsd/freebsd
+curl, https://github.com/curl/curl.git
+tcl, https://github.com/tcltk/tcl.git
+cpython, https://github.com/python/cpython.git
+nginx, https://github.com/nginx/nginx.git
+qemu, https://github.com/qemu/qemu.git
+apr, https://github.com/apache/apr.git
+tmux, https://github.com/tmux/tmux.git
+php-src, https://github.com/php/php-src.git
+gtk, https://gitlab.gnome.org/GNOME/gtk.git
+gstreamer, https://gitlab.freedesktop.org/gstreamer/gstreamer.git
+openssh-portable, https://github.com/openssh/openssh-portable.git
+openssl, https://github.com/openssl/openssl.git
+wireshark, https://github.com/wireshark/wireshark.git
+git, https://github.com/git/git.git
+linux, git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
+gmp,https://github.com/ryepdx/gmp
+lighttpd1.4,https://github.com/lighttpd/lighttpd1.4.git
+lighttpd2,https://github.com/lighttpd/lighttpd2.git

python/data/defects4j-bugs.csv

@@ -0,0 +1,396 @@
+project,bugId,repairActions,repairPatterns,repairTools,files,methods,classes,linesAdd,linesRem,linesMod,sizeInLines,chunks,spreadAllLines,spreadCodeOnly,mdAdd,mdRem,mdRen,mdParAdd,mdParRem,mdRetTyChange,mdParTyChange,mdModChange,mdOverride,mcAdd,mcRem,mcRepl,mcParSwap,mcParAdd,mcParRem,mcParValChange,mcMove,objInstAdd,objInstRem,objInstMod,varAdd,varRem,varTyChange,varModChange,varReplVar,varReplMc,exTryCatchAdd,exTryCatchRem,exThrowsAdd,exThrowsRem,condExpRed,condExpExpand,condExpMod,condBranIfAdd,condBranIfElseAdd,condBranElseAdd,condBranCaseAdd,condBranRem,assignAdd,assignRem,assignExpChange,loopAdd,loopRem,loopCondChange,loopInitChange,tyAdd,tyImpInterf,retExpChange,retBranchAdd,retRem,wrapsIf,wrapsIfElse,wrapsElse,wrapsTryCatch,wrapsMethod,wrapsLoop,unwrapIfElse,unwrapMethod,unwrapTryCatch,condBlockExcAdd,condBlockRetAdd,condBlockOthersAdd,condBlockRem,copyPaste,missNullCheckP,missNullCheckN,expLogicExpand,expLogicReduce,expLogicMod,expArithMod,codeMove,wrongVarRef,wrongMethodRef,singleLine,constChange,notClassified,blockRemove,fixAPI,wrongComp,missComp,others,initFix,size1,size2,size3,size4,size5,size6,size7,size8,size9,size10,size10+,Observations,rtAcs,rtKali,rtDynaMoth,rtGenProg,rtDeepRepair,rtCardumen,rtssFix,rtHDRepair,rtElixir,rtJAID,rtGPFL,rtNopol
+Chart,1,1,2,7,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,,Change Operator,,X,X,X,X,X,,,,X,X,
+Chart,2,4,2,0,1,2,1,14,0,0,14,8,493,318,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,
+Chart,3,1,0,6,1,1,1,2,0,0,2,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,x,,,,,,,,,,Missing initialization,,,,X,X,X,,,X,,X,X
+Chart,4,1,2,3,1,1,1,2,0,0,2,2,8,8,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,X,,,X,,,,,,X
+Chart,5,4,2,7,1,1,1,4,0,1,5,2,4,2,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,X,X,X,X,X,,,,,X,X
+Chart,6,8,1,1,1,1,1,7,0,1,8,1,0,0,,,,,,,,,,x,x,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,x,,,,,,x,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,X,,,,,,
+Chart,7,3,3,4,1,1,1,0,0,2,2,2,1,1,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,X,X,X,,,,,X,
+Chart,8,2,2,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,x,x,,,,,,,,,,x,Changes params passed to constructor.,,,,,,,,X,X,,,
+Chart,9,1,2,6,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,X,,X,X,,,X,X,,X
+Chart,10,2,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,
+Chart,11,2,2,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,x,,,,x,,,,,,,,,,,Replaces var referencing of target object of a call.,,,,,,X,,,X,,,
+Chart,12,3,3,3,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,x,,,,,,,,,,,"It involves a ""wrap with method"", but the real change is converting from a class attribute assignment to a ""set"" method call.",,,,,X,X,,,,,X,
+Chart,13,6,3,7,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,x,,x,,x,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,x,,,,,,x,,x,x,,,,,,,,,,x,,,X,X,X,X,X,,,X,,X,
+Chart,14,2,3,2,2,4,2,12,0,0,12,4,518,243,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,X,,,,X,,,,,,,
+Chart,15,2,4,6,1,2,1,5,0,0,5,3,675,212,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,x,,,,,,,,,,x,,,,x,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,X,X,X,X,X,,,,,X,
+Chart,16,5,2,0,1,2,1,0,0,3,3,2,129,59,,,,,,,,,,x,,,,,,,,x,,,,,,,,x,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,x,,,x,,,,,,,,,Variable initialization in constructor + conditional expression change.,,,,,,,,,,,,
+Chart,17,5,0,4,1,1,1,1,0,1,2,1,0,0,,,,,,,,,,x,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,x,,,,,,,,,,Type change + computing change.,,,X,,,X,,,X,,,X
+Chart,18,9,4,2,2,3,2,10,2,1,13,6,19,9,,,,,,,,,,x,x,,,,,,,x,,,x,,,,,,,,x,,,,,x,,,,x,x,,,,,,,,,,,x,x,,,,,,x,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Replaces return point by throw exception.,,,,,X,,,,,,X,
+Chart,19,3,3,1,1,2,1,6,0,0,6,2,275,117,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,X,,,,,,,,,,,
+Chart,20,2,2,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,x,x,,,,,,,,,,,Replaces params value passed to constructor.,,,,,,,X,,,,,
+Chart,21,6,2,1,1,2,1,39,0,0,39,4,585,239,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,x,,,x,,,x,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,It seems that the functionality wasn't implemented before.,,,,,,,,,,,,X
+Chart,22,11,4,0,1,3,1,30,1,2,33,7,145,72,,,,,,,,,,x,,,,,,x,,x,,,x,,,,x,,,,x,,,,,x,,,,x,x,,,x,,,,,,x,,,,,,,,,x,,,x,,x,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,Implicit null check after loop throught a vector and using a flag variable.,,,,,,,,,,,,
+Chart,23,7,1,0,1,1,1,19,0,0,19,1,0,0,x,,,,,,,,x,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,"I don't know if it makes sense to describe everything made in a method definition... specially in cases like that, where many lines of code with specific logic are commited.",,,,,,,,,,,,
+Chart,24,2,3,3,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,x,,,,,x,,,,x,,,,,,,,,,,Replaces variable in an assignment expression.,,,,,,X,X,,,X,,
+Chart,25,7,5,7,1,2,1,12,0,2,14,6,226,185,,,,,,,,,,x,x,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,x,,,,,,,,x,,x,,,,,,,,,,x,,,x,x,x,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,X,X,X,X,X,,,,,X,X
+Chart,26,1,2,6,1,1,1,2,0,0,2,2,5,5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,X,X,,X,X,,,,X,,X
+Closure,1,2,1,2,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,X,,,,,,,,,X
+Closure,2,3,2,1,1,1,1,4,0,0,4,3,3,1,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,X
+Closure,3,10,2,2,1,2,2,6,0,2,8,3,218,119,,,,x,,,,,,x,,x,,x,,,,,,,x,,,,,,,,,,,,x,x,,,x,,x,,,,,,,,,,x,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,X,,,,,,,,,X
+Closure,4,2,3,0,1,1,1,0,0,2,2,2,11,9,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,
+Closure,5,3,1,3,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,X,,,,,,,X,,X
+Closure,6,2,2,0,1,2,1,0,8,0,8,4,36,20,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,
+Closure,7,5,2,2,1,1,1,4,0,1,5,3,2,0,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,x,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,X,,,,,,X
+Closure,8,4,1,2,1,2,1,4,0,0,4,2,4,3,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,Adds a new condition in expression and a new method invocated by this condition.,,,X,,,,,,,,,X
+Closure,9,5,1,0,1,2,2,1,0,1,2,2,64,43,,,,,,,,,,x,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,,,,,,,,,,
+Closure,10,2,2,4,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,x,,,,,,,,,,x,,,,X,,,X,,X,,,,X
+Closure,11,3,1,0,1,1,1,0,2,0,2,1,0,0,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,
+Closure,12,6,1,3,1,1,1,6,0,0,6,1,0,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,x,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,X,,,X,,,,,,X
+Closure,13,1,2,2,1,1,1,1,1,0,2,2,1,1,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,x,,,,,,,,,,Only one line of code was moved.,,,X,,,X,,,,,,
+Closure,14,2,3,3,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,x,,,,,,,,x,,,,,,,,,,,Parameter value change.,,,,,,,X,X,,,,X
+Closure,15,3,1,1,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,X
+Closure,16,8,1,2,1,3,2,6,0,3,9,5,296,217,,,,x,,,,,,x,,x,,x,,x,,,,x,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,x,,,,,,X,,,,,,,,,X
+Closure,17,5,2,2,1,1,1,4,0,1,5,2,1,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,x,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,Transform If structure in Wrap-with IfElse.,,,X,,,,,,,,,X
+Closure,18,1,2,3,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,x,,,,,,,,,,x,,,,X,,,,,,,X,,X
+Closure,19,1,1,1,1,1,1,2,0,0,2,2,1,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,X,,,,,,,,,
+Closure,20,2,3,2,1,1,1,1,0,1,2,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,X,,,,,,,,,X
+Closure,21,8,3,3,1,1,1,0,17,2,19,2,11,7,,,,,,,,,,x,x,,,,,,,,,,,x,,,,,,,,,,x,,,,,,x,,x,,,x,,,,,,,x,,,,,,,,,,,,,x,,,,x,,x,,,,,,,,,,x,,,,,,,,,,,,,,,,,,X,,,X,,,,,,X
+Closure,22,8,4,3,1,1,1,0,24,2,26,5,12,8,,,,,,,,,,x,x,,,,,,,,,,,x,,,,,,,,,,,,x,,,,x,,x,,,x,,,,,,,x,,,,,,,x,,,,,,x,,,x,,,x,,,,,,,,,,x,,,,,,,,,,,,,,x,"It seems that the old conditions were wrong, so a replacement was made. It simplifies too the implementation. It is similar to a Wrap-with-If operation plus a move of some conditionals.",,,X,,,X,,,,,,X
+Closure,23,5,2,0,1,1,1,6,0,1,7,2,1,1,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,x,,,,,x,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,
+Closure,24,4,2,0,1,1,1,2,1,2,5,4,15,9,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,x,,,,,,,x,,,,,,,"Again, a kind of Wrap-with-IfElse where some conditionals were missing.",,,,,,,,,,,,
+Closure,25,5,0,0,1,1,1,2,4,0,6,4,22,18,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,
+Closure,26,5,1,2,1,1,1,5,0,0,5,3,100,71,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,X,,,,,,,,,X
+Closure,27,6,2,0,1,3,1,3,0,3,6,3,120,99,x,,,,,,,,,,,x,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,x,,,,,,,,,x,,,,,,A Node object is instantiated in the call. Adds new method and replace calls.,,,,,,,,,,,,
+Closure,28,3,0,2,1,1,1,4,0,0,4,1,0,0,x,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,x,,,,,,,x,Overrides the addConstant method. Replaces method calls.,,,X,,,,,,,,,X
+Closure,29,6,2,2,1,1,1,10,0,0,10,3,59,28,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,x,,,,x,,,,,,,,,,x,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,X,,,,,,,,,X
+Closure,30,9,5,2,2,3,3,8,0,2,10,6,358,237,,,,,,,,,,,x,x,,x,,,,,,,x,,,,,,,,,,x,,,x,x,,,,x,,,,,,,,,,x,,,x,,,,,,,,,x,,,,x,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,x,x,Adds a new class attribute with its initialization.,,,X,,,,,,,,,X
+Closure,31,1,2,4,1,1,1,0,1,0,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,X,,,,X,,,X,,X
+Closure,32,8,2,0,1,1,1,15,3,0,18,6,56,38,,,,,,,,,,x,x,,,,,,,,,,x,,,,,,,,,,,,,x,x,,,x,x,,,x,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,
+Closure,33,3,1,4,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,X,,,X,,,,X,,X
+Closure,34,5,3,0,2,3,2,0,2,3,5,3,629,505,,,,,,,,,,x,,x,,x,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,x,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,
+Closure,35,7,2,1,1,1,1,0,13,2,15,1,0,0,,,,,,,,,,x,x,,,,,,,,,,,x,,,,,,,,,x,,,,,,,x,,x,,,x,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,x,Simplifies an existent null check. It's a kind of conditional block replacement.,,,,,,,,,,,,X
+Closure,36,3,2,2,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,X,,,,,,,,,X
+Closure,37,5,2,0,2,2,2,4,0,1,5,3,3,0,,,,,,,,,,x,,x,,x,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,x,Replaces method call by overloaded method with extra parameters.,,,,,,,,,,,,
+Closure,38,1,2,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,"Parentesis added has no effect on expression evaluation, so a conditional expression expansion was considered only.",,,,,,,,,,,,X
+Closure,39,5,3,0,1,1,1,0,0,3,3,3,12,9,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,x,,,,,,,,,"If-else implicit in ""exp?a:b"" construct",,,,,,,,,,,,
+Closure,40,2,2,5,1,1,1,0,2,1,3,2,2,2,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,,,,,,,"Removed a null check, maintaining the branch code.",,,X,,,X,X,,,X,,X
+Closure,41,4,2,0,1,2,1,9,0,0,9,2,191,138,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,
+Closure,42,3,1,0,1,1,1,7,0,0,7,2,2,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,
+Closure,43,6,4,0,1,2,1,17,0,0,17,4,161,108,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,x,,,,x,,,x,,,,,,,,,,x,,,,,,,,,,x,,,x,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,
+Closure,44,2,1,0,1,1,1,2,0,0,2,2,5,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,
+Closure,45,8,3,3,1,2,2,4,0,2,6,4,171,107,,,,,,,,,,x,x,,,,,,,,,,x,,,,,,,,,,,x,x,x,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,x,,,,,,,,,x,,,,,,x,,,,,,,,,X,,,X,,,,,,X
+Closure,46,9,1,2,1,1,1,0,16,0,16,1,0,0,,x,,,,,,,x,,x,,,,,,,,x,,,x,,,,,,,,,,,,,,,,x,,x,,,x,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,x,Removes Overriding method. Adds and initializes local variable.,,,X,,,X,,,,,,
+Closure,47,5,1,0,2,2,2,5,0,3,8,3,3,2,,,,,,,,,,,,,,,,x,,,,x,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,
+Closure,48,6,2,1,1,1,1,3,0,4,7,2,1,0,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,x,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,X,,,,,,,,,
+Closure,49,10,2,2,1,3,1,14,8,0,22,8,81,53,,,,,,,,,,x,x,,,,,,,,,,x,x,,,,,,,,,,,,,,,x,x,x,x,,x,x,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,X,,,,,,,,,X
+Closure,50,3,3,2,1,1,1,5,0,1,6,3,13,9,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,X,,,,,,,,,X
+Closure,51,2,2,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,X,,,,
+Closure,52,2,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,
+Closure,53,4,1,0,1,1,1,4,0,0,4,2,19,13,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,
+Closure,54,9,5,0,2,3,2,10,1,2,13,5,31,18,,,,,,,,,,x,x,,,,,,,,,,x,,,,x,,,,,,x,,,x,x,,,,x,,x,,,,,,,,,,,x,,,,,,,,,,x,,,,x,,x,,,,x,,,,,,,,x,,,,,,,,,,,,,,One of the chunks have a kind of Wrap-with-IfElse done with the replacement of existent conditional block.,,,,,,,,,,,,
+Closure,55,2,2,3,1,1,1,1,0,1,2,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,x,,,,,,,,,x,,,,X,,,X,,,,,,X
+Closure,56,3,1,0,1,1,1,4,0,0,4,2,1,1,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,
+Closure,57,2,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,"Altought there is a null check in the conditional expression, the expression is expanded with an extra condition not related to null-check.",,,,,,,,,,,,
+Closure,58,2,1,0,1,1,1,4,0,0,4,2,2,2,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,
+Closure,59,2,2,1,1,1,1,1,0,1,2,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,X,,,,,,,,,
+Closure,60,4,2,2,1,2,1,5,0,0,5,3,29,17,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,x,,x,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,X,,,,,,,,,X
+Closure,61,5,2,2,1,1,1,6,0,0,6,1,0,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,x,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,X,,,,,,,,,X
+Closure,62,1,2,4,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,,"Just changed the operator in ""if"" condition",,,X,,,,,X,,X,,X
+Closure,63,1,2,3,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,,Same as bug 62,,,X,,,,,,,X,,X
+Closure,64,5,1,0,1,3,1,0,0,4,4,4,38,25,,,,x,,,,,,,,x,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,New parameters are added in three method calls (call to overloaded methods). A logic expression passed to a method call is changed.,,,,,,,,,,,,
+Closure,65,1,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,
+Closure,66,2,1,2,1,1,1,2,0,0,2,2,1,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,X,,,,,,,,,X
+Closure,67,2,2,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,X,,,,,,,,,X
+Closure,68,1,3,0,1,3,1,3,1,0,4,4,892,554,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,x,,,,,,,x,An assignment was moved outside a branch.,,,,,,,,,,,,
+Closure,69,2,1,0,1,1,1,7,0,0,7,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,x,,,,,,,,,,,,,
+Closure,70,1,2,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,x,,,,,,,,,,x,,,,,,,,X,,,X,,
+Closure,71,3,3,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,,The null-check is done in a boolean expression assigned to a variable and not to a conditional test.,,,,,,,,,,,,
+Closure,72,3,1,2,2,2,2,2,0,1,3,2,0,0,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,Instantiates an anonymous object and make a method call on it.,,,X,,,,,,,,,X
+Closure,73,1,2,3,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,X,X,,X,,
+Closure,74,9,3,0,1,2,1,13,0,2,15,2,165,136,x,,,,,,,,,x,,x,,,,,,,,,x,,,,,,,,,,,,,x,,,x,,x,,x,,,,,,,,x,,,,,,,,,,,,x,,,x,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,
+Closure,75,5,3,2,1,2,1,3,0,1,4,3,62,46,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,x,x,,,,,,,,,,,,x,,,,,,,,,,,x,,,x,,,,,,,,,,,x,,,,,,,,,,,X,,,,,,,,,X
+Closure,76,9,4,2,1,2,1,37,6,0,43,11,73,48,,,,,,,,,,x,x,,,,,,,,,,x,,,,,,,,,,,,x,x,x,,,x,x,,,,,,,,,,x,,,,,,,,x,,,,x,x,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,X,,,,,,,,,X
+Closure,77,2,2,0,1,1,1,1,0,0,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,
+Closure,78,1,1,2,1,1,1,0,2,0,2,2,6,6,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,Removes error invocation call.,,,X,,,,,,,,,X
+Closure,79,3,1,0,2,2,2,1,0,1,2,2,0,0,,,,,,,,,,x,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,x,,,,,,,,,,Calls overloaded method with extra parameters.,,,,,,,,,,,,
+Closure,80,1,1,0,1,2,1,2,0,0,2,2,1648,989,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,"These blocks are inserted as extra switch clauses. Should be a ""Missing Condition"", ""Missing Block"" or both?",,,,,,,,,,,,
+Closure,81,4,1,0,1,1,1,7,0,0,7,1,0,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,
+Closure,82,2,2,0,1,1,1,2,0,1,3,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,
+Closure,83,2,1,0,1,1,1,3,0,1,4,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,"The existent code was splitted in declaration of var ""param"" out of the wrap and assignment, with Wrap-with-TryCatch.",,,,,,,,,,,,
+Closure,84,7,2,0,1,3,1,27,0,0,27,3,465,377,x,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,x,,x,,,,,,,,,,x,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,
+Closure,85,9,4,0,1,2,1,7,13,1,21,4,29,22,,,,,,,,,,x,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,x,x,x,,x,,,,,,,,x,,,,,,,,x,,,,x,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Closure,86,1,3,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,x,x,,,,,,,,x,,,,,,,,,,x,,,,,,,,X,,,,,
+Closure,87,8,1,0,1,1,1,12,0,1,13,4,10,0,,,,,,,,,,x,x,,,,,,,,,,x,,,,,,,,,,,,,x,x,,,,x,,,,,,,,,x,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Closure,88,5,1,0,1,1,1,6,0,0,6,2,3,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,
+Closure,89,3,3,0,2,2,2,3,0,1,4,2,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,x,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,
+Closure,90,4,2,0,2,2,2,3,0,1,4,2,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,,
+Closure,91,5,2,0,1,1,1,9,0,0,9,1,0,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,x,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,x,,,,,,,,,,,,,
+Closure,92,2,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,
+Closure,93,2,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,x,,,,,,,,,,,Same change/commit as #92.,,,,,,,,,,,,
+Closure,94,3,2,0,1,1,1,19,0,0,19,3,8,6,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,"Many ""case"" conditions were added, sharing the same consequent action. Equivalent to ""if ( x || y || z || ... ) then do this"". These blocks are inserted as extra switch clauses. Should be a ""Missing Condition"", ""Missing Block"" or both?",,,,,,,,,,,,
+Closure,95,4,1,0,1,1,1,7,0,0,7,2,3,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,
+Closure,96,3,3,0,1,1,1,3,0,1,4,3,3,1,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,
+Closure,97,3,2,0,1,1,1,1,0,1,2,1,0,0,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,x,,,,x,,,,,,,,,,,,,,,,,,,,,,
+Closure,98,6,3,0,1,2,2,19,0,0,19,5,201,127,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,x,,,x,,,,,,,x,,,,,,,,,,,,x,x,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,New class attributes were added.,,,,,,,,,,,,
+Closure,99,6,3,0,1,1,1,5,0,1,6,4,39,26,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,x,,x,,,,x,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,x,,,,,,,x,,,,,x,"Changed condition in expression, adds an extra computing in line 131.",,,,,,,,,,,,
+Closure,100,6,2,0,1,1,1,7,0,1,8,2,47,34,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,x,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,
+Closure,101,2,1,0,1,1,1,1,3,0,4,2,1,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,
+Closure,102,1,2,0,1,1,1,1,1,0,2,2,5,5,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,
+Closure,103,5,4,0,2,2,2,8,0,0,8,3,3,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,x,x,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,
+Closure,104,2,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,,A Null-Check was removed and replaced by another condition.,,,,,,,,,,,,
+Closure,105,7,3,0,1,1,1,2,0,5,7,5,19,16,,,,,,,,,,,x,,,,,,,x,x,,,,,,,,,,,,,,x,,x,,,,x,,x,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,x,,,,,,,,,,,,,x,,,,,,,x,,,,,Moves the instantiation of StringBuilder to another place.,,,,,,,,,,,,
+Closure,106,4,3,0,2,2,2,4,2,0,6,3,1,1,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,x,,,,,,,,x,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,
+Closure,107,1,1,0,1,1,1,1,0,0,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,x,,,,,,,,,,x,,,,,,,,,,,,,
+Closure,108,4,1,0,1,3,1,3,0,1,4,4,322,230,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,x,,,,x,,,,,,,,Added new class attribute and its initialization. An existent conditional expression with Null-Check was expanded with another condition.,,,,,,,,,,,,
+Closure,109,5,3,0,1,1,1,4,0,1,5,1,0,0,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,x,,,x,,,,,,,,,x,,,,,,,,,,,,x,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,,,,
+Closure,110,12,7,0,2,2,2,21,0,4,25,12,47,22,,,,,,,,,,x,x,,,,,,,x,,,x,,,,,,,,x,,,x,x,x,x,,,,x,,x,,,,,,,x,,,,x,,,,,,,,x,,x,,,x,x,x,,x,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,
+Closure,111,3,2,1,1,1,1,1,0,1,2,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,X
+Closure,112,6,1,0,1,1,1,6,0,3,9,1,0,0,x,,,,,,,,x,x,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,Adds an inline interface implementation for a parameter of type Predicate on filterKeys method call.,,,,,,,,,,,,
+Closure,113,2,2,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,"Again, an existent condition with Null-Check was expanded with other condition.",,,,,,,,,,,,X
+Closure,114,2,2,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,X
+Closure,115,5,1,1,1,1,1,0,11,0,11,2,25,15,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,X
+Closure,116,5,2,2,1,2,1,12,0,0,12,2,26,15,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,x,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,Strange... what is removed in 115 is added again in 116.,,,X,,,,,,,,,X
+Closure,117,9,1,0,1,1,1,12,12,0,24,3,31,20,,,,,,,,,,,,,,,,,x,,,,x,x,,,,,,,,,,,,x,,,,x,x,x,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,Move a block of code.,,,,,,,,,,,,
+Closure,118,2,1,0,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,
+Closure,119,1,2,1,1,1,1,1,0,0,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,"""Case"" Added to switch... Should be considered a ""missing block"" too?",,,,,,,,,,,,X
+Closure,120,3,1,1,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,X
+Closure,121,2,2,1,1,1,1,2,0,1,3,2,3,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,X
+Closure,122,5,1,0,1,1,1,1,0,1,2,1,0,0,,,,,,,,,,x,x,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,
+Closure,123,3,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,x,,,x,x,,,,,,,,,,,Changes value in assignment from constant to method call.,,,,,,,,,,,,
+Closure,124,2,1,2,1,1,1,2,0,0,2,2,1,1,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,X,,,,,,,,,X
+Closure,125,2,2,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,x,"Again, conditional expression with Null-Check was expanded with new condition.",,,,,,,,,,X,,
+Closure,126,4,1,5,1,1,1,0,4,0,4,1,0,0,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,X,,,,X,X,,X,,X
+Closure,127,5,3,2,1,2,1,9,0,1,10,2,6,5,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,x,,x,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,X,,,,,,,,,X
+Closure,128,3,2,0,1,1,1,3,0,1,4,2,6,6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,x,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,
+Closure,129,3,0,2,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,x,,,,,,,,,Adds a loop to iterate over child of nodes.,,,X,,,,,,,,,X
+Closure,130,1,2,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,x,,,,X,,,,,,,,,X
+Closure,131,2,2,2,1,1,1,1,0,1,2,2,5,4,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,X,,,,,,,,,X
+Closure,132,2,2,2,1,1,1,2,0,1,3,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,X,,,,,,,,,X
+Closure,133,1,1,3,1,1,1,1,0,0,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,x,,,,,,,,,,,It seems that the variable is reseted by method.,,,X,,,X,,,,,,X
+Lang,1,6,3,0,1,1,1,9,0,2,11,3,3,3,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,x,,,x,,,,x,,,x,,,,,,,,,,,,,,,,,,,,x,,x,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,
+Lang,2,4,1,0,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,
+Lang,3,1,2,0,1,1,1,4,0,0,4,4,12,11,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,
+Lang,4,5,1,0,1,1,1,0,0,4,4,4,43,27,,,,,,,,,,x,,,,,,,,,,x,,,x,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,x,,,,x,,,,,,,,Type change with adjust in related calls/operations.,,,,,,,,,,,,
+Lang,5,6,3,0,1,1,1,21,0,0,21,2,31,31,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,x,,,,,,x,,,,x,,,,,,,,,,x,,,x,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,
+Lang,6,2,3,3,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,X,X,X,,,
+Lang,7,6,2,4,1,2,1,3,3,0,6,3,270,167,,,,,,,,,,x,x,,,,,,,x,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,"In one of the chunks, removes a conditional block with return.",X,,,,X,X,,,,,X,
+Lang,8,5,0,0,1,2,1,1,2,0,3,3,34,14,,,,,,,,,,x,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,x,,,x,,,,,,,,,Transformed a class attribute in local var.,,,,,,,,,,,,
+Lang,9,4,1,0,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,
+Lang,10,4,1,2,1,1,1,0,9,0,9,2,2,2,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,,,,,,X,X,,,,,,
+Lang,11,4,2,0,1,1,1,4,0,0,4,1,0,0,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,
+Lang,12,5,3,0,1,1,1,7,0,0,7,3,8,7,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,x,x,,,,x,,,,,,,,,,,,,x,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,
+Lang,13,8,3,0,1,2,1,19,0,0,19,4,30,12,,,,,,,,,,x,,,,,,,,x,,,x,,,,,,x,,x,,,,,,x,,,,x,,,,,,,,,,x,,,,,x,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,
+Lang,14,3,1,1,1,1,1,3,0,0,3,2,1,1,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,X,,,,,,
+Lang,15,4,2,0,1,1,1,0,0,4,4,2,453,226,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,x,,,,,x,,,,,,,x,Change local var type.,,,,,,,,,,,,
+Lang,16,2,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,
+Lang,17,6,2,0,1,1,1,2,8,1,11,6,11,9,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,x,x,x,,,,,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,x,x,x,,,,,,,,,,,,"Unwraps input.length() from ""Character.codePointCount()"" method. Unwraps ""for loop"" from else branch. Unwraps ""pos += ..."" from if-else branch.",,,,,,,,,,,,
+Lang,18,3,2,0,1,1,1,2,2,1,5,2,1,1,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,An existent if-else structure is removed and a new one is placed in an action that resembles a wrap-with replacement. An aditional wrap-with-IfElse is present in IfElse-CondExp form.,,,,,,,,,,,,
+Lang,19,9,5,0,1,1,1,6,0,3,9,5,38,29,,,,,,,,,,x,x,,,,,,,,,,x,,,,,,,,,,,,x,x,x,,,,x,,,,,x,,,,x,,,,x,,,,,,,,,x,,,,,,,,x,x,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,
+Lang,20,4,2,1,1,2,1,0,0,2,2,2,84,27,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,"Null-Check is removed from ""buf"" initialization.",,,,,X,,,,,,,
+Lang,21,3,4,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,x,,x,x,,,,,,,,x,,,,,,,,,,,,,,,,,,X,,,,,
+Lang,22,5,3,3,1,1,1,6,0,1,7,2,1,0,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,x,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,X,X,,,,,X,
+Lang,23,6,1,0,1,2,1,31,0,0,31,3,196,105,x,,,,,,,,x,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,Implements equals and hascode to work with hash structures of Collections,,,,,,,,,,,,
+Lang,24,1,2,6,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,X,,,,X,X,,,X,X,X,
+Lang,25,1,1,0,1,1,1,0,0,27,27,1,0,0,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,x,Changes keys in a vector of type key-value.,,,,,,,,,,,,
+Lang,26,1,2,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,x,x,,,,,,,,,,,Adds a new parameter to constructor call (call overloaded constructor),,,,,,,,,X,,,
+Lang,27,4,2,2,1,1,1,3,0,1,4,2,9,9,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,X,X,,,,,,
+Lang,28,3,1,0,1,1,1,6,0,0,6,2,1,1,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,
+Lang,29,1,1,0,1,1,1,0,0,1,1,1,0,0,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,x,Changes return type of method.,,,,,,,,,,,,
+Lang,30,11,7,0,1,6,1,38,0,5,43,20,305,102,,,,,,,x,,,x,x,,,,,,,,,,x,,,,,,,,,,x,x,x,x,x,,,,x,,,,,,,,,,x,,x,x,,,,,,,,,x,x,,x,,,x,,x,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,
+Lang,31,4,2,0,1,1,1,8,0,0,8,4,7,5,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,,Partially the same as Lang 30,,,,,,,,,,,,
+Lang,32,7,4,0,1,3,1,14,5,3,22,4,427,74,,x,,,,,,,x,x,,,,,,,,,,x,x,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,x,x,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,Applies null check using synchronized block with object lock (HashCodeBuilder.class),,,,,,,,,,,,
+Lang,33,1,3,3,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,Wrap-with applied with IfElse-Expression,,,,,,,X,,X,X,,
+Lang,34,3,3,0,1,2,1,0,0,2,2,2,15,3,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,x,x,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,
+Lang,35,4,0,1,1,2,1,0,0,2,2,2,278,62,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,"Changes return of a null object to throw an Exception. The only statement of an existent ""else block"" is replaced by an exception throw.",X,,,,,,,,,,,
+Lang,36,3,2,0,1,1,1,8,0,1,9,2,896,471,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,
+Lang,37,7,2,0,1,1,1,8,0,0,8,3,2,1,,,,,,,,,,x,,,,,,,,x,,,x,,,,,,x,,x,,,,,x,,,,,x,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,
+Lang,38,1,1,4,1,1,1,1,0,0,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,x,,x,,,,,,,,,,,,,,,,X,,,,X,X,X,
+Lang,39,1,2,5,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,x,,,,,,X,X,,,X,X,X,
+Lang,40,7,1,0,1,1,1,7,0,1,8,1,0,0,,,,,,,,,,x,x,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,x,,,,,,x,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,
+Lang,41,8,2,0,1,2,1,19,0,2,21,8,58,23,,,,,,,,,,x,,,,,,,,x,,,x,,,,,,,,,,,x,,x,,,,,x,,,x,,,,,,x,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,,Existent condittional expression with null-check is expanded.,,,,,,,,,,,,
+Lang,42,6,2,0,1,1,1,5,0,2,7,2,2,2,,,,,,,,,,x,,x,,,,,,,,,,,x,,,,,,,,,,,,x,,,,x,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,,,,,
+Lang,43,1,1,3,1,1,1,1,0,0,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,x,,,,,,,,,,,,,,,,,,X,X,X,,,
+Lang,44,4,1,3,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,X,,,,,,X,,,X
+Lang,45,3,1,1,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,X,,
+Lang,46,5,1,0,1,6,1,3,0,7,10,9,152,82,,,,x,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,,,,,
+Lang,47,2,3,0,1,2,1,6,0,0,6,2,44,21,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,x,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,
+Lang,48,3,1,0,1,1,1,4,0,0,4,2,2,1,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,
+Lang,49,4,2,0,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,,,,,,x,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,
+Lang,50,8,4,0,1,2,1,2,6,4,12,6,179,68,,,,,,,,,,x,x,,,,,,,x,x,,,,,,,,,,,,,,x,,,,,x,x,x,,,,,,,,,,,,,,,,,x,,,,,,x,x,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,,,"Replaces some existent Null-Chek, inverting postive by negative.",,,,,,,,,,,,
+Lang,51,1,1,4,1,1,1,1,0,0,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,"Adds a return statement that does the role of an ""else"" branch.",,,X,,,,,,X,X,,X
+Lang,52,2,1,0,1,1,1,4,0,0,4,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,
+Lang,53,0,3,2,1,1,1,2,2,0,4,4,10,8,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,x,,,,,,,,,,x,,,,,x,,,,,,,,,,,X,,,,,,,,,X
+Lang,54,4,1,0,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,
+Lang,55,1,1,2,1,1,1,2,0,0,2,2,1,1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,X,,,,,,,X,,
+Lang,56,4,0,0,1,1,1,4,0,2,6,3,880,425,x,,,,,,,,x,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,x,,,,,,Changes types modifiers and overrides method related to Serialization.,,,,,,,,,,,,
+Lang,57,3,2,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,x,,,,,,,,,,,Changes direct acces of an object to method call.,,,,,,,,X,X,,,
+Lang,58,1,1,3,1,1,1,0,1,1,2,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,,X,,,,,,X,,,X
+Lang,59,2,2,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,x,,,,x,,,,,,,,,,,,,,,,,,X,,X,,,
+Lang,60,2,2,0,1,2,1,0,0,2,2,2,56,21,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,
+Lang,61,2,3,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,x,,,,,x,,,,x,,,,,,,,,,,Changes/fix computing of a variable.,,,,,,,,,,X,,
+Lang,62,2,2,0,1,1,1,8,0,0,8,4,76,61,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,x,"Added a ""break"" in ""case"" branch.",,,,,,,,,,,,
+Lang,63,8,1,1,1,2,1,2,19,1,22,4,118,91,,x,,,,,,,,x,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,x,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,x,x,,,,,,,,,,,,,,,,,,X,,,,,,,,,
+Lang,64,9,3,0,1,2,1,21,0,0,21,5,12,2,x,,,,,,,,,x,,,,,,,,x,,,x,,,,,,x,,x,,,,,x,,,,,x,,,,,,,,,,x,,,,,,,,,,,x,x,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,
+Lang,65,5,3,0,1,1,1,30,0,0,30,8,86,50,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,x,,x,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,
+Math,1,2,2,0,2,2,2,6,0,0,6,2,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,x,,,,,,,,,,,,,
+Math,2,1,2,8,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,x,,,,,,,,,,,Apply a casting to returned value of an expression.,,X,X,X,X,X,,,X,,X,X
+Math,3,2,1,1,1,1,1,3,0,0,3,2,1,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,X,,,,,,,,,,,
+Math,4,2,3,2,2,2,2,6,0,0,6,2,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,x,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,"Almost a copy, the only difference is in involved variable.",X,,,,,,,,,,,X
+Math,5,1,3,9,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,x,x,,,,,,,,x,,,,,,,,,,,,X,,X,X,X,X,,X,X,X,X,
+Math,6,8,4,2,7,7,7,2,4,12,18,17,327,240,,,,,,,,,,x,,,,,,x,,,,x,,x,,,,x,,,,,,,x,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,x,,,x,,,,x,,,x,,,,,,,,,,,,A parameter value passed through the constructor during object instantiation is changed. A local variable is removed and expressions to increment it are replaced by equivalent method call.,,,,,X,X,,,,,,
+Math,7,6,3,3,1,1,1,5,6,3,14,4,18,11,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,x,x,,x,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,Changes involving adding and removing of many loops.,,,,X,X,,,,,,,X
+Math,8,3,0,5,1,1,1,0,0,2,2,2,5,4,,,,,,x,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,Ajust the object initialization to reflect type changed.,,X,,X,X,X,,,,,X,
+Math,9,4,1,0,1,1,1,1,0,1,2,1,0,0,,,,,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,x,,,,,,,,,,Replaced constructor by overloaded version. Changes object attribute value by direct assignment.,,,,,,,,,,,,
+Math,10,2,1,0,1,1,1,1,0,0,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,
+Math,11,2,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,
+Math,12,3,0,0,1,0,1,3,0,1,4,3,11,3,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,,,,,,imports and implements java.io.Serializable.,,,,,,,,,,,,
+Math,13,7,1,0,1,1,1,9,0,0,9,2,2,2,,,,,,,,,,x,,,,,,,,x,,,x,,,,,,,,,,,,,,x,,,,x,,,x,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,Contains the patch of Math-13,,,,,,,,,,,,
+Math,14,10,1,0,2,2,2,9,3,1,13,3,2,2,,,,,,,,,,x,x,,,,,,,x,,,x,,,,,,,,,,,,,,x,,,,x,x,,x,x,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,
+Math,15,3,1,0,1,1,1,1,0,1,2,2,1228,742,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,
+Math,16,5,2,0,1,2,1,19,0,2,21,11,377,235,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,x,,,,,,,,,,x,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,
+Math,17,3,1,0,1,1,1,4,0,0,4,2,1,1,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,x,,,,,,,,,,,,,
+Math,18,5,2,2,1,3,1,2,0,4,6,5,60,38,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,X,X,,,,,,
+Math,19,7,1,0,1,1,1,13,0,0,13,1,0,0,,,,,,,,,,x,,,,,,,,x,,,x,,,,,,,,x,,,,,x,,,,,x,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,
+Math,20,4,3,4,1,1,1,1,0,1,2,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,x,,,x,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,x,,,x,,,,,,,,,,"Adds a new return output alternative through conditional expression of the form ""exp?a:b"". Something to think: ""Is it a change in return expression, a return branch adding or both?"". For now, it is considered as both.",,,,,X,X,,,X,,X,
+Math,21,6,3,0,1,1,1,4,1,9,14,6,50,31,,,,,,,,,,,,,,,,,,,,,x,x,,,x,,,,,,,,,x,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,x,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,
+Math,22,1,1,2,2,2,2,0,0,2,2,2,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,x,,,,,,X,,,X,,,,
+Math,23,5,1,0,1,1,1,7,0,2,9,5,124,104,,,,,,,,,,x,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,Call overloaded method with extra params. New variable is added with its initialization.,,,,,,,,,,,,
+Math,24,2,2,3,1,1,1,0,0,2,2,2,36,34,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,X,,X,,,,,,,X
+Math,25,3,1,1,1,1,1,3,0,0,3,2,2,0,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,X,,,,,,,,,,,
+Math,26,2,2,0,1,1,1,0,0,2,2,2,27,21,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,
+Math,27,2,3,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,x,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,
+Math,28,2,1,7,1,1,1,4,0,0,4,4,33,23,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,X,X,X,X,X,,,,,X,X
+Math,29,10,1,0,1,2,1,12,1,3,16,2,21,12,,,,,,,,,,x,x,,,,,,,,,,x,x,,,,,,,,,,,,x,x,,,,x,x,,x,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,x,x,,,,,,,,,,,,,Changed a while loop to a for lor loop.,,,,,,,,,,,,
+Math,30,1,1,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,X,,,X,,,
+Math,31,11,4,0,1,1,1,1,21,12,34,4,23,17,,,,,,,,,,x,x,,,,,,,,x,,x,x,,,x,,,,,x,,,,,,,,x,x,x,,,x,,,,,,,,,,,,,,,,,,,x,x,,,,,,,x,,x,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Math,32,2,3,6,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,X,X,,X,X,,,X,X,,
+Math,33,2,3,5,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,x,,,,,,,,,x,,,,,,,,,,x,A variable used in a method call is replaced by another one.,,,X,,,X,X,,X,,,X
+Math,34,2,2,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,x,,,x,,,,,,,,,,x,A direct class attribute access was replaced by a method call that gives indirect access to the object.,,,,,,,,X,X,,,
+Math,35,3,3,1,1,2,1,0,0,2,2,2,13,3,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,Similar to Math-34,X,,,,,,,,,,,
+Math,36,4,2,0,1,2,1,12,0,0,12,4,49,17,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,"The change is almost the same in two different places. The only difference between changes is the use of the call ""doubleValue()"" in place of ""floatValue()"" in correspondent methods.",,,,,,,,,,,,
+Math,37,4,3,0,1,2,1,12,0,2,14,4,46,11,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,x,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,x,"The changes are almost the same in two different places... same structure, but with variattions in the parameters/arguments.",,,,,,,,,,,,
+Math,38,3,2,1,1,1,1,0,0,4,4,3,89,70,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,x,,,x,,,,x,,,,,,,,Removes Exception Thrown. The throws line is commented in code... maybe the change isn't a final one.,,,,,,,,,,,,X
+Math,39,3,1,1,1,1,1,9,0,0,9,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,x,,,,,,,,,,,,,X
+Math,40,3,2,7,1,1,1,6,0,2,8,2,2,1,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,x,,,,,,,,,,,x,,,,,,X,X,X,X,X,,,,,X,X
+Math,41,4,2,3,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,x,x,,,,,,,,,,,The changes were made only inside the for loop.,,,X,,,X,,,,,,X
+Math,42,1,2,2,1,1,1,2,0,1,3,2,3,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,X,,,,,,,,,X
+Math,43,1,1,0,1,1,1,0,0,3,3,3,4,4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,
+Math,44,5,1,4,1,1,1,6,1,0,7,3,62,39,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,x,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,The variable and assignment adding are part of the for loop.,,,X,,X,,,,,,X,X
+Math,45,5,1,0,1,1,1,5,0,0,5,1,0,0,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,x,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,
+Math,46,2,2,2,1,1,1,0,0,2,2,2,36,23,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,"The removed if-else condition is in the form ""exp?a:b"".",,,X,,,X,,,,,,
+Math,47,5,3,0,1,1,1,2,0,3,5,4,209,80,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,x,,,,x,,,,,,,,,x,,,,x,,,,,,,,,,,,x,,,,,x,,,,,,,,,,x,,,,,,,,x,,,,,,,"The ""WrapWith"" condition is in the form ""exp?a:b"". The added variable is a class attribute.",,,,,,,,,,,,
+Math,48,3,1,0,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,
+Math,49,2,2,7,1,4,1,0,0,4,4,4,35,29,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,Changed the target of a iterator access.,,X,X,X,X,X,,,,,X,X
+Math,50,3,1,11,1,1,1,0,4,0,4,1,0,0,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,X,X,X,X,X,X,X,X,X,X,X
+Math,51,7,1,0,1,1,1,9,0,0,9,3,4,1,,,,,,,,,,x,,,,,,,,x,,,x,,,,,,,,x,,,,,x,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,Adds an exception throw in default clause of switch-case.,,,,,,,,,,,,
+Math,52,4,1,0,1,1,1,1,0,3,4,3,13,7,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,x,,,,,,,,New variable is added with its initialization.,,,,,,,,,,,,
+Math,53,2,1,6,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,X,X,,X,X,,X,X,
+Math,54,7,2,0,1,1,1,6,0,1,7,2,2046,1332,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,x,x,x,,,,x,,,,,,,,,,x,,,x,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,,
+Math,55,8,2,0,1,1,1,17,0,1,18,5,14,0,,,,,,,,,,x,,,,,,x,,,,x,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,x,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,
+Math,56,4,1,2,1,1,1,0,6,1,7,1,0,0,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,x,,,,,,,,,,x,,,,,Replaces a wrong and more complex calculation by the correct and simpler one.,,,,,X,,,,,,X,
+Math,57,1,1,6,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,Just changed the declared type of a primitive variable.,,,X,,X,X,X,,X,,,X
+Math,58,4,2,6,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,x,,,x,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,x,,,,,,,,,,,Call overloaded method without one parameter.,,,X,,X,X,X,,X,,,X
+Math,59,2,2,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,X,,,
+Math,60,8,3,3,1,1,1,2,9,1,12,2,2,2,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,x,,,,x,x,,,x,,,,,,,,,,,x,x,,,,,,,,,x,,x,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,Unwraps try-catch.,,,,X,X,X,,,,,,
+Math,61,2,0,1,1,1,1,0,0,2,2,2,71,18,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,x,,,,,,,,,,Replaces Exception Thrown. Changed the type of exception thrown. Changed the imported type.,X,,,,,,,,,,,
+Math,62,13,2,1,1,1,1,0,1,3,4,2,13,9,,,,,,,,,,x,x,x,,x,,x,,,,,x,x,,,,,,,,,,,,,x,,,x,x,x,x,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,x,,,,,,,x,,,,,,,,"Changes the way to compute ""optima[i]"", changing params for call to ""optimize"". Removed if-branch is in the form ""exp?a:b"".",,,,,,X,,,,,,
+Math,63,3,2,3,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,X,X,,,X,,,
+Math,64,15,7,1,1,1,1,19,5,3,27,13,180,131,,,,,,,,,,x,x,,,,,x,x,x,x,,x,,,,x,,,,,,,,,x,,,,x,x,,x,x,,,,,,,x,x,,,,,,,,,,,x,x,x,,,x,,,,x,x,x,,,,,x,,x,,,,,,,,,,,,,,,Moved instantiation of object of type VectorialPointValuePair to another place (lines ahead).,,,,,X,,,,,,,
+Math,65,7,2,0,1,1,1,0,5,2,7,2,12,5,,,,,,,,,,x,,,,,,x,,,,,,x,,,,x,,,,,,,,,,,,,,x,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,
+Math,66,19,4,0,1,4,1,0,11,11,22,12,178,129,,x,,,x,,,,x,x,x,x,,,x,x,,,x,,x,x,,,,x,,,,x,,,x,,x,,,,x,,,,,x,,,,x,,x,,,,,,,,,,,,,,,,,,,x,,,x,x,,x,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,
+Math,67,2,1,0,1,2,1,0,0,2,2,2,4,2,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,A direct access to a vector was added in place of a value returned from a call to an object method. The change was similar in structure in two different places.,,,,,,,,,,,,
+Math,68,9,3,0,1,1,1,10,0,2,12,9,253,160,,,,,,,,,,x,,,,,,,,x,x,,x,,,,,,,,,,,,,x,x,,,,x,,,,,,,,,x,x,,,x,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,New objects were instantiated. Objects instantiated before were removed too (line 419).,,,,,,,,,,,,
+Math,69,2,2,3,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,x,,,,x,,,,,,,,,,,Fix assignment operation.,,,X,,,X,,,,,,X
+Math,70,3,2,7,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,x,,,,,,,,,,,Call overloaded method without one parameter.,,,,X,X,X,X,X,X,,X,
+Math,71,2,1,4,2,2,2,7,0,0,7,2,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,Three lines equal lines are added in two different places. Just line 302 is different.,,,X,X,X,,,,,,X,
+Math,72,2,1,0,1,1,1,0,0,2,2,2,11,7,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,x,,,,,,,,,,"Similar changes in two different places, the same melhod is targeted, the same argument is changed, only the new values applied are different.",,,,,,,,,,,,
+Math,73,3,1,6,1,1,1,4,0,0,4,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,X,X,X,,,X,,X,X
+Math,74,5,0,2,1,1,1,3,0,4,7,3,2,2,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,x,,,,,,,x,,,,,A vector/object was instantiated.The way to initialize values was changed.,,,,,X,X,,,,,,
+Math,75,2,2,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,X,,,
+Math,76,7,5,0,1,1,1,6,0,8,14,12,95,67,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,x,,,,x,x,x,,,x,,,,,,,,x,,,,,,,,,,,,x,,,,,x,x,,x,,,,,,,,,,,,,,,,,,,,,,The structure of the changes are similar and applied in two different methods.,,,,,,,,,,,,
+Math,77,8,1,1,2,2,2,0,9,1,10,2,0,0,,x,,,,,,,x,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,x,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,x,,,,,,,,,,,,,x,,"Fix a assignment. Removes a method (aparently, unrelated changes).",,,,,X,,,,,,,
+Math,78,6,2,7,1,1,1,11,0,0,11,3,8,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,x,,,,,x,,,,,x,,,x,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,X,X,X,X,X,,,,,X,X
+Math,79,1,0,2,1,1,1,0,0,2,2,2,1,1,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,X,X,,,,,
+Math,80,1,2,9,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,x,,x,,,,,,,,,,,Fix expression in assignment (added parathesis).,,X,X,X,X,X,,,X,X,X,X
+Math,81,4,3,6,1,3,1,1,0,3,4,3,938,643,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,x,,,,,x,,,,,,x,,,,,,,,Fix/updates assignment to var upperSpectra.,,X,,X,X,X,,,,,X,X
+Math,82,1,2,10,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,,Changes operator in condition.,X,X,X,X,X,X,,X,X,X,X,
+Math,83,3,2,0,1,2,1,0,0,3,3,3,51,34,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,"The added ""WrapWith"" uses ""exp?a:b"" format.",,,,,,,,,,,,
+Math,84,6,1,5,1,1,1,9,0,0,9,3,30,15,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,x,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,X,,X,X,X,,,,,X,
+Math,85,1,2,11,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,,Changes operator in condition.,X,X,X,X,X,X,,X,X,X,X,X
+Math,86,6,1,0,1,1,1,3,3,0,6,2,20,14,,,,,,,,,,,,,,,,,,x,x,,,,,,,,,,x,x,,,,x,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,The condition to throw the exception is moved from one place to another.,,,,,,,,,,,,
+Math,87,2,1,2,1,1,1,0,2,2,4,3,2,2,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,X,,,,,,,,,X
+Math,88,8,2,3,1,1,1,5,6,0,11,4,7,5,,,,,,,,,,x,,,,,,,,x,,,x,,,,,,,,,,,,,,x,,,x,x,x,,,x,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,X,,,X,,,,,,X
+Math,89,3,1,1,1,1,1,4,0,0,4,2,1,1,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,X,,,,,,,,,,,
+Math,90,2,0,1,1,2,1,3,0,0,3,2,11,0,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,Added a new overloaded method reusing part of the implementation of an existing one. The old method just call the new one.,X,,,,,,,,,,,
+Math,91,3,1,0,1,1,1,0,0,2,2,1,0,0,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,x,,x,,,,,,,,,,The value of a class attribute is accessed in place of a method to do the assignment.,,,,,,,,,,,,
+Math,92,15,6,0,1,2,1,48,5,1,54,12,92,19,,,,,,,,,,x,x,,,,,x,,x,x,,x,,,,,,,,x,x,,,,x,x,,,x,x,,x,x,,,,,,,x,,,,,,,,,,,x,x,x,x,,,,,,,x,,,x,,,,x,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,
+Math,93,7,3,1,1,2,1,8,0,2,10,3,50,15,,,,,,,,,,x,,,,,,,,x,,,,x,,,,,,,x,,,,x,x,,,,,,,,,,,,,,,x,,,,,,,,,,,x,x,,,,,,,,x,,,,,,,,,,x,x,,,,,,,,,,,,x,,,X,,,,,,,,,,,
+Math,94,1,2,1,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,X,,,,,
+Math,95,2,1,4,1,1,1,2,0,1,3,3,3,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,X,,X,,X,,,,,X,
+Math,96,2,3,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,,Changes logical expression in assignment,,,,,,,,,,,,
+Math,97,4,3,2,1,1,1,14,0,2,16,5,9,5,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,x,,,,,,,,,,,,,x,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,X,,,X,,,,,,
+Math,98,1,2,1,2,2,2,0,0,2,2,2,0,0,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,x,,x,,,,,,,,,,Changes dimension of two vectors.,,,,,X,,,,,,,
+Math,99,3,1,1,1,2,1,8,0,0,8,2,171,68,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,Instantiates a vector object as part of the exception creation.,X,,,,,,,,,,,
+Math,100,2,2,0,1,2,1,0,0,3,3,3,39,26,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,Apply the same replace of methods in three different places.,,,,,,,,,,,,
+Math,101,2,2,2,1,1,1,1,0,1,2,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,X,,,X,,,,,,
+Math,102,6,2,0,1,1,1,17,0,0,17,3,5,5,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,x,,,,x,,,x,,,,,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,
+Math,103,3,3,0,1,1,1,10,0,0,10,2,2,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,x,,,,,x,,,,,,x,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,,
+Math,104,1,2,2,1,0,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,,,,x,x,,,,,,,,,,,,,,,,,X,,,X,,,
+Math,105,2,2,4,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,X,,,X,,,,X,,X
+Math,106,3,2,0,1,1,1,8,0,0,8,4,36,18,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,Only one variable is different between the two blocks of code in patch.,,,,,,,,,,,,
+Mockito,1,6,1,0,1,1,1,11,0,1,12,1,0,0,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,x,,,,x,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,
+Mockito,2,4,1,0,1,2,1,7,0,0,7,3,25,13,x,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,import and instantiate a new Reporter object. It appears as an validation object/procedure.,,,,,,,,,,,,
+Mockito,3,9,1,0,1,2,1,15,1,2,18,3,13,12,x,,,,,,,,,x,x,,,,,,,x,,,x,,,,,,,,,,,,,,x,,,,x,,,x,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Mockito,4,3,4,0,1,3,1,3,0,2,5,3,251,223,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,x,,,,,,x,,,x,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,
+Mockito,5,1,1,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,Change parameter type in catch command.,,,,,,,,,,,,
+Mockito,6,3,3,0,1,20,1,0,0,20,20,20,320,41,,,,,,,,,,,,,,,,x,,x,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,Instantiates specific objects in place of call do Any.ANY,,,,,,,,,,,,
+Mockito,7,2,1,0,1,1,1,1,0,0,1,1,0,0,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,x,,,,,,,,,,,Instantiates and initializes an TypeVariable vector.,,,,,,,,,,,,
+Mockito,8,1,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,"Replaced and ""else"" with an ""else if""",,,,,,,,,,,,
+Mockito,9,4,1,0,1,1,1,5,0,0,5,3,28,5,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,
+Mockito,10,7,0,0,1,4,1,5,0,5,10,7,81,36,x,,,x,,,,,,,,x,,x,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,x,,"Changes signature of existent methods, adding new params.",,,,,,,,,,,,
+Mockito,11,7,2,0,1,2,1,8,0,1,9,3,5,4,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,x,,,,x,,,,,,,,,x,x,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,
+Mockito,12,3,1,0,1,1,1,4,0,0,4,3,2,1,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,
+Mockito,13,3,2,0,1,1,1,2,0,1,3,3,5,3,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,
+Mockito,14,4,2,0,2,2,2,4,0,1,5,5,109,80,,,,,,,,,,x,,,,,,x,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,"An existing parameter (""mode"") was wrapped by a constructor call, instantiating MockAwareVerificationMode object",,,,,,,,,,,,
+Mockito,15,3,1,0,1,1,1,3,0,0,3,3,22,11,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,Instantiated a BeanPropertySetter object,,,,,,,,,,,,
+Mockito,16,5,1,0,2,3,2,2,1,3,6,5,73,6,,x,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,Merged two overloaded method definitions.,,,,,,,,,,,,
+Mockito,17,6,2,0,2,3,2,7,0,2,9,6,80,61,,,,,,,,,,x,x,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,
+Mockito,18,3,1,0,1,1,1,2,0,0,2,1,0,0,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,Returns instantiated ArrayList<Object>,,,,,,,,,,,,
+Mockito,19,11,2,0,5,5,5,18,0,8,26,11,41,20,x,,,x,,,,,,x,,x,,x,,,,x,,,,,,,,,,,,,,,,x,x,,,,,,,x,,,,,,x,x,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Adds new parameter to existent method. Adjust code to reflect this.,,,,,,,,,,,,
+Mockito,20,7,1,0,1,1,1,1,0,2,3,3,12,11,,,,,,,,,,x,,x,,,,,,x,,,x,,,,x,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,Instantiates an InstantiatonProvider object,,,,,,,,,,,,
+Mockito,21,14,2,0,1,3,1,16,0,4,20,5,15,10,x,,x,x,,,,x,,x,,x,,,,,,,,,x,,,,,,,,x,,,,,x,,,,,x,,,x,,,,,,x,x,x,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,"Apply ""static"" modifier to withParams method",,,,,,,,,,,,
+Mockito,22,2,1,0,1,1,1,2,0,1,3,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,
+Mockito,23,11,2,0,1,5,2,15,0,3,18,8,77,48,x,,,,,,,,,x,,,,,,,,x,x,,,,x,x,,,,,,,,,,x,,,,,x,x,,,,,,x,,x,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Instantiates MockitoCore and ReturnsEmptyValues objects. Replaces instantiation of Answer to SerializabelAnswer object.,,,,,,,,,,,,
+Mockito,24,3,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,"If-Else implicit in ""exp?x:y"" format",,,,,,,,,,,,
+Mockito,25,13,0,0,1,6,1,25,2,4,31,10,71,29,x,,,x,x,,,,x,x,x,x,,x,,x,,x,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,
+Mockito,26,1,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,x,,,,x,,,,x,,,,,,,,,,,,,,,,,,,,,,,
+Mockito,27,5,0,0,1,1,1,0,1,1,2,1,0,0,,,,,,,,,,x,x,,,,,,,,x,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,
+Mockito,28,3,2,0,1,1,1,1,0,1,2,1,0,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,x,,,,,,,,,,Changes method signature adding new parameter.,,,,,,,,,,,,
+Mockito,29,1,3,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,"Wrapping implicit in ""exp?x:y"" format",,,X,,,,,,,,,X
+Mockito,30,4,0,0,2,2,2,1,0,2,3,3,3,3,,,,x,,,,,,,,x,,x,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,x,,,,,,,,,"Adds new parameter to method signature, ajust code.",,,,,,,,,,,,
+Mockito,31,4,0,0,1,1,1,2,0,1,3,2,51,25,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,
+Mockito,32,2,1,0,1,1,1,4,0,1,5,2,30,25,,,,,,,,,,,,x,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,
+Mockito,33,7,3,0,1,1,1,11,0,1,12,2,1,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,x,,,,,,,x,x,,,,,,,,,,,x,x,,,,x,,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,
+Mockito,34,2,2,0,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,
+Mockito,35,3,2,0,1,3,1,0,0,3,3,3,152,31,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,
+Mockito,36,3,1,0,1,1,1,4,0,0,4,2,191,146,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,Instantiates an Reporter object,,,,,,,,,,,,
+Mockito,37,3,1,0,1,2,1,8,0,0,8,2,2,1,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,
+Mockito,38,1,3,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,X,,,,,,,,,X
+Time,1,7,4,0,2,2,2,9,3,1,13,3,4,4,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,x,,,x,,,x,,,x,,,,,,,,,,,,x,,,,,,,,,,x,,x,x,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Time,2,5,4,0,2,2,2,6,0,1,7,3,230,108,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,x,,,x,,,,,,,,,,,,,,,x,,,,,,,,,,,,x,x,,,x,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,
+Time,3,1,2,0,1,10,1,20,0,0,20,20,242,70,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"The patches makes a type of ""bypass"" null check, where the logic is applyed only if the value is different of null.",,,,,,,,,,,,
+Time,4,4,2,7,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,x,,,x,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,x,x,,,,,,,,,,,The swap was made on a constructor call.,,X,X,X,,X,,,X,,X,X
+Time,5,12,3,0,1,1,1,6,0,6,12,2,1,1,,,,,,,,,,x,x,,,,,x,,x,,,x,,,,x,,,,x,,,,x,x,,,,,x,x,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,x,,,,,x,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,
+Time,6,7,3,0,1,3,2,26,0,0,26,3,805,556,,,,,,,,,,x,,,,,,,,x,,,x,,,,,,,,x,,,,,x,x,,,,x,,,,,,,,,,,,,,,,,,,,,x,,x,,x,,,,,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,,
+Time,7,4,0,3,1,1,1,1,1,0,2,2,2,2,,,,,,,,,,,x,x,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,x,,x,,,,,,,,,,Move assignment of var and changed assignment expression.,,,X,,,X,,,,,,X
+Time,8,6,5,0,1,1,1,3,0,2,5,3,6,6,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,x,,,,x,x,,,,,,,x,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,x,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,
+Time,9,5,2,1,1,2,1,6,0,3,9,5,22,15,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,x,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,X,,,,,,
+Time,10,3,1,0,1,1,1,1,0,1,2,2,52,25,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,x,,,,,,,,,,Adds new variable and its initialization.,,,,,,,,,,,,
+Time,11,5,0,6,1,1,1,1,0,4,5,1,0,0,x,,,,,,,,x,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,,x,,,,,,,Changed a static initialization for a method inline implementation.,,X,X,X,,X,,,X,,,X
+Time,12,8,3,2,2,2,2,12,0,2,14,8,70,26,,,,,,,,,,x,,,,,,x,,x,,,x,,,,,,,,,,,,,x,x,,,,x,,,,,,,,,,x,,,x,,,,,,,,,x,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,The if/else added is implicit in the passed parameter format (exp?x:y) for the LocalDate constructor call.,,,,,,,,,,,X,X
+Time,13,6,3,0,1,1,1,4,0,1,5,3,43,37,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,x,,,,x,,x,,,,,,,,,,,x,,,,,,,,,,x,,,,,,,,x,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,
+Time,14,5,1,2,1,1,1,5,0,0,5,2,1,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,x,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,X,,,,,,,,,X
+Time,15,3,1,2,1,1,1,3,0,0,3,1,0,0,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,X,,,,,,,,X,,,
+Time,16,3,2,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,x,,,,,,x,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,x,,,,,,,,x,x,,,,,,,,,,,Changes parameter for constructor call.,,,X,,,,,,,,,X
+Time,17,11,4,1,1,1,1,13,0,5,18,5,8,2,,,,,,,,,,x,x,,,,,,,,,,x,x,,,,,,,,,,,x,x,x,,,,x,,x,,,,,,,,x,x,,,,,,,,x,,,x,,,,,,,,x,x,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,,,X,,,,,,
+Time,18,5,2,2,1,1,1,12,0,0,12,2,3,3,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,x,,,,,x,,,,,x,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,X,,,,,,X
+Time,19,1,2,2,1,1,1,0,0,1,1,1,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,,X,,,,X
+Time,20,7,4,0,1,1,1,6,0,2,8,3,4,4,,,,,,,,,,x,x,,,,,,,,,,x,,,,,,,,,,,,,x,,,,,x,,,,,,,,,,x,x,,,,,,,,,,,x,x,,,x,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,
+Time,21,8,4,0,1,1,1,12,2,4,18,5,10,6,,,,,,,,,,x,x,,,,,x,,,,,x,,,,,,,,,,,,x,x,,,,,x,,,x,,,,,,,,,x,,,,,,,,,,,x,,,,x,,,,,,x,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Time,22,4,0,0,1,1,1,5,0,1,6,2,1,0,,,,,,,,,,x,,x,,,,,,,,,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,x,,,x,,,,,,x,,,,,,,,,,,,,,,,,,
+Time,23,3,1,0,1,1,1,5,4,4,13,8,17,17,,,,,,,,,,x,x,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,x,,,,,,,,,,,,Changes entries in a key-value map.,,,,,,,,,,,,
+Time,24,4,1,0,1,1,1,5,0,0,5,1,0,0,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,x,,,x,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,x,,,,,,,,,,,,,,,,,,,
+Time,25,6,2,0,1,1,1,9,0,0,9,1,0,0,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,x,x,,,,x,,,,,,,,,,x,,,,,,,,,,,,x,x,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,x,,,,,,,,,,,,,,,
+Time,26,3,2,0,1,7,1,0,0,7,7,7,98,84,,,,,,,,,,,,x,,x,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,x,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,
+Time,27,1,2,0,1,1,1,2,0,0,2,2,3,3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,x,,,,,,,,,,,,,,,,,,,,,,

python/data/defects4jCommits13April.txt

@@ -0,0 +1,369 @@
+commons-math 0da657a65c92b086a301a6ffe9e34ec272f8889c 1
+commons-math c0b655ace5665c0cd32e3f5e5b46edad4d223125 2
+commons-math 92c4697300e8ddf06152bc0838c738d50673c1d2 3
+commons-math 277e61721f34be16a20da663fd597edf6b51939b 4
+commons-math e54a1c92302f3167b06bc04859b87ac0681bcdf3 5
+commons-math 419a052c6842192e78f747d9f5af619c2ca56e78 6
+commons-math 1cd68eed57febf806c385bf04a596b922f4c1964 7
+commons-math 196e63174afdb3df1529c1b97bb8437b16831cc3 8
+commons-math de98c0f0a566060ac143c39036f06a03f141dc52 9
+commons-math 7e2ffcc9034de41d7787f0b33b5670474f7a10de 10
+commons-math aa2bd1d0656b0001192aa2b2ef779cfd1f3b7e4d 11
+commons-math cc82d38aaf6ec24758a0a6df9535b8204e22ef12 12
+commons-math 8079ea5b8d1366445da532906e43afa9291473cf 13
+commons-math a6b2e992e17cee0d4cb5a2da8242a20b8e5a8fc3 14
+commons-math b221a7342856a2a548f5d9817055defc0ec4dea6 15
+commons-math 875ed1f4d90d1457c0ab40dafc79be5a0c6f9bf6 16
+commons-math 621806b796bc416f00341feca894ebae07be5ed0 17
+commons-math 7c7d7e8f103582e753c39a2baf14a483e991fefb 18
+commons-math c73fad0a0d42103b5e13a68317ea95b1090263ba 19
+commons-math efa9de05114492ca38cf4739a07339f5ad6faddc 20
+commons-math 468ed8550b7759858a4dc59d694859a8ae7d35f0 21
+commons-math b0cba9a79ee99b09958dec8e40c75bed47b7f780 22
+commons-math f1b04e990f4fc86544b6230fab1aa8ecfb74fbb1 23
+commons-math d5ff460ba69e4261f066d7856e2f90b886924513 24
+commons-math 996dd998e3081e4a842017b8ebcdae9b6059b530 25
+commons-math 1566dd339f6efc2347b0962fac7fce22adbc31ff 26
+commons-math 5f2077bb774d283e7984a2d5dc0c2759f2954963 27
+commons-math d3fc5af31eb696af03cfbe2e18584c7e1d307d54 28
+commons-math 7bbddc2203bed78fafe7739a97df1f53e767341a 29
+commons-math a25e7f7abe7f6b3f4147febee4a917ce92241aab 30
+commons-math 2d846db3aec18dd081e680be05f6e0faad1cb186 31
+commons-math f81847d8079753ce45b049813730055188a22efb 32
+commons-math 3ef79313597d7d49067c48d65a09042d8d94822d 33
+commons-math 520f36b43a13681db338a07ac6c251cbe1a7a6e5 34
+commons-math a5d8d115cb85149c62059a23e8f4d318cac0e903 35
+commons-math 1a15d5f4c13eca0435b0ed7e6a624064e7f7e07f 36
+commons-math 65ed08e15af15617e967d3ea9d635dc55a0ef866 37
+commons-math 91cc42ba0493938aa53585720b315b62c5784a96 38
+commons-math 74e00296574dc3ac0bc064fc3258faabaf732d6c 39
+commons-math c8becc7c47963bcdc1578298846ad6fbf08f64ef 40
+commons-math 882556eabbeb2f62939aee29afdec2a01ce4bbe1 41
+commons-math e98a5000cd211539bf4ba65f62cc7f81395e1726 42
+commons-math dcae84b2e8f025e93340307d8bc04d406202c323 43
+commons-math 53c8cec5ceb4cd81e3f9b8858814accac83a324e 44
+commons-math bc4e9db01c2a03062965fa4bac65782376ab2287 45
+commons-math e6f27ebcb9ee0a344308382b99a3894bb61b225d 46
+commons-math 330f3fe17d132bd4e2a91ff812ccf489e77f390f 47
+commons-math 13d22f45624470ce5c07c085cf8b1ec0251eaee6 48
+commons-math 09e35881f1ad74335707e70aa78fd347e37d1066 49
+commons-math 39cf5e69259d7560d50553caf028f9229b721013 50
+commons-math 2f066a5b2d2fe8a00a251a3220b0d52446fe392d 51
+commons-math 3c319db494928e9d51ea6091b301302c65f4eceb 52
+commons-math 7707b0bb80be05bbf6533a36bb0c646cbfd1026d 53
+commons-math c00ac8120a4215125e49775cd9351e689586b467 54
+commons-math 0c38504ffa3ef520ce78e260538d7b8742c895e8 55
+commons-math e06fe05e2dd68936e770ba67caa7b9924568170d 56
+commons-math 00fea9d8078d487e31cec8292dbd9bd69bc9c216 57
+commons-math 45add3a0e7e2e94bfc29e85c9ef0856e2e473a33 58
+commons-math 5dcca48038fb6274cc155251d09db12746ccce71 59
+commons-math 9054aac4b948117b838d6a5b15be1f50965d805a 60
+commons-math f76efe5f4ef36fadc677c94269927076f2f42eb9 61
+commons-math 8de3eb542b2be80ba309b2c91ae1dba75bdb5063 62
+commons-math d2a5bc02c002acfa220ce8bad23b9e8af137f47d 63
+commons-math 7dadc2ab019f066a7a287376ad4c63193e8a0a9a 64
+commons-math a4f9188a55e6935d4d38ae97806af4d59e671588 65
+commons-math 76fcbc838c0f27d9b029c1f283390cb4f47f8895 66
+commons-math fc505bba569dede0d29a401798fa9f55df9e27f1 67
+commons-math 615ca9a000c253575e6f62bed87db6110b750834 68
+commons-math a2711c189d9754c22e3aba2de1c6c125e52626aa 69
+commons-math f41fcd85ca62a2109a6e550be0353d292d351213 70
+commons-math 6537e18ad189603e19615226e6aa1a9cdd154b8f 71
+commons-math aa09ac7ca6e01a2ba41470ca590f07f2ada8af6f 72
+commons-math 29c3b75e2d5120771ca85ffcbffc9b7cb5d45c58 73
+commons-math d06e7b7c8e646ff489a5dfba47dfcb283a194e9b 74
+commons-math 523df0c13719150b1397a6094f410274bfaf1475 75
+commons-math 39e2ad8af8fdedac51144158061cb31e7863c3a1 76
+commons-math d6555de715889237b7be11639d164e7098862003 77
+commons-math b3ed2415fc58b34667d99f077bae6c8e38b7aced 78
+commons-math 138baec1c778c2ea1dc1a6efe6d5e76a3b072b1d 79
+commons-math 3ced4f2a4e2546f0d7c309bd8f6305edee0dee8f 80
+commons-math 7d903ca533833e5db8d2fe150f1e31b54ddda700 81
+commons-math 4ece1884cc408e15c8f1db9010fec5cac43937b9 82
+commons-math f36be8e8b7371fb8814456b6c095866aa802de78 83
+commons-math 317793eeff84dc9b260301adbe4aa8f5c79f703a 84
+commons-math 9be86f674f91fa3fe0a8694e7b98472dabe6886d 85
+commons-math bd8f05c4dda4b5b00e525e08fd6c95d256423c97 86
+commons-math 75f5c92aeb47e264c196a8c38a495adac89f493c 87
+commons-math 24a6a2692e942969f5c39bd88d3d12ac0f0bf0d9 88
+commons-math 62b3877f953dd47c4d301be35c77446e2cf55311 89
+commons-math 0c84b28c3542f3340f7ec0dffc34c60dff66604e 90
+commons-math e1df5f5486f10ac521dfc3dc20d1bfe508e67aa8 91
+commons-math 85a8ab2dcfc26e83be1514acdb11d3feaac3e73d 92
+commons-math 7cd3d822b65b7acdd1f6e1c82b291cd994f4fe05 93
+commons-math a96e597894562c2cf01fc678f6c435d65ccc31c5 94
+commons-math fbf87122e0f7229892b6dbbf2e211cc46acea008 95
+commons-math e6449cccdeaba96ffba4c27db322c5c3d5c18662 96
+commons-math ed492bd0c5c5c3a0258a65cb31cc8723d8f011fd 97
+commons-math a0f3e0435c405ec842240413e2165863c9c76a16 98
+commons-math 58b5609fe8f99aabc990d885bf6c4d85631b7c79 99
+commons-math 876d133334e8dde309cc11f884c0dd4cc9ce530e 100
+commons-math 32643d19538ad853a1280eb4060c4f15ac6dc3dd 101
+commons-math a1bac127067c912b4a9f7a2957c427853a36c730 102
+commons-math 4ce05bcd51ec956d789d20b59c743603d24a8ab7 103
+commons-math e39a7750ef362679599b623b05bfadb150832515 104
+commons-math fc21b26f84312e4f75e8b144238618c73a8b091f 105
+commons-math 41ba9e00e3bbde990f6821f67f0da2a5575b9ac3 106
+commons-lang 687b2e62b7c6e81cd9d5c872b7fa9cc8fd3f1509 1
+commons-lang 09d39029b16dee61022dc8afde4d84f523f37813 2
+commons-lang 2c9c8753165dc7ce5dd1d5a6d741b445b33302c2 3
+commons-lang fb47b96ab635d7cc6e9edefdddc46f1baf63b117 4
+commons-lang 75944e541d358d5b06ebbba3098a919fbf2539d4 5
+commons-lang cff0f1ae37bb2b7ab2dcdb10dec1f3cad7532e1c 6
+commons-lang e71f6dd3f2f70c640ae73d28b432b3a69ffcab4b 7
+commons-lang a4a455053e7e4a77320efd6db0814f4de82d754d 8
+commons-lang c45d5bff75edfef6387d40e681f9dc7c9b517bbb 9
+commons-lang afe5dff79fc23d222ba1f1481fbbf55e09d2e6da 10
+commons-lang c821fafce06adefc040e8983fb12d49399a48627 11
+commons-lang c9d786a4fe2639581047d878a64bab6fb24f1ab3 12
+commons-lang bd59a1908a1df3364918be7b07782500616b15ac 13
+commons-lang cf7211f9d7d70d56501d8c4c827bf9ce3cac5f0b 14
+commons-lang 8185a9e681d6de5f40b0ed07b40730c9761bad37 15
+commons-lang 13c7f19a5ad506340a7c8d8601ef4bf2426ab325 16
+commons-lang b4255e6d071dc8d48bfc51037cecb29097b9a49d 17
+commons-lang 2aa9dca994b006ccbfcb528de5ff0da6a5578411 18
+commons-lang 0b3a8a3128890f710700d04e38df48b883d11d91 19
+commons-lang f08213cc4c35fd71e98aa5233fcb2db3333793a2 20
+commons-lang 55f642725742513824975af0e5e91cb1920b6d72 21
+commons-lang 0b2b82ad923b0455cde93286e3f504a1abd0863b 22
+commons-lang 46356183872f2cba5e340993c6d85597a4f3cdbb 23
+commons-lang 81903abbb25ff92597d5f1e3acc648d12bdee444 24
+commons-lang 2bae68787d15c3d3540380164e62759485078ec1 25
+commons-lang 75d5b74a1a7ef68c5169f4b44cdf972c1ab07ee0 26
+commons-lang c2560c028173c3bc0a4627d4ce5ce221bcc9be66 27
+commons-lang 3e1afecc200d7e3be9537c95b7cf52a7c5031300 28
+commons-lang 982e295053663787bb0396b81a8956c3c87dc25b 29
+commons-lang 19b6372018c6b800f8e2bf1b85e15ef4cbd500da 30
+commons-lang 38bf7048571f3bc8dd34d14691906426444e4381 31
+commons-lang 006fca88e86bd6f650d4d021d2ff3573a572827d 32
+commons-lang 0603aef594fa60126c2d45f2ab164eee39f7b44c 33
+commons-lang 496525b0d626dd5049528cdef61d71681154b660 34
+commons-lang 80bd78ea0685d28ddec9b8162d652db9dec949ce 35
+commons-lang da0612b348fcfa7679b92a5e9b8e1603e8e2da3a 36
+commons-lang ea140fb5c327e2b58f6c5bf1057c7dede909a50c 37
+commons-lang a95e76eefb91bdd9cf1c22734874a3d3c95ed327 38
+commons-lang cb40e35f5e0990fad4c5278964fcc24e112dde8c 39
+commons-lang 8b1a8e178abb46cc048a3982636b8ec4e6ffc8dc 40
+commons-lang 4d3629f310e84cc24c0e5f09d97f5126692d0128 41
+commons-lang dd2ae757d12554f290931f0b3c05e33ad993ecd1 42
+commons-lang eb3e2ae1f3734986a9f0225c661888baac3cb13b 43
+commons-lang 81f10d7f1ef52d44f43b38d81ac3dc1c83d45134 44
+commons-lang d95fcd8e24568b4f9d3cb9da922a4029e293c793 45
+commons-lang 916639bda234a8e5a030361c4068667764b34cb7 46
+commons-lang d5e343049c2610c31ddacb24f0d173942f6d5ac9 47
+commons-lang eb8f74efb75e71fc91e515a38becc2aac203e339 48
+commons-lang 0ac772a4e3b07287c435fdc3c812f67277c85cc8 49
+commons-lang 659ef247b1452b6d5b9f92271357381fa59fea50 50
+commons-lang a5589b39e985e64861bd920aa7b13ab32c215b06 51
+commons-lang 868f6976aea222e96259843f56df9aed8e48584b 52
+commons-lang d3b1163073c7aeb6011a5b65bfadad15bfc0f108 53
+commons-lang e89e8d62b911340cc5b293465cdae909f3dfd640 54
+commons-lang 3ef8a796b5ddf87e3e9afec9ba94aac62681b394 55
+commons-lang 672e3a8ead2970181c32cf5fc70a3ea3d020c1fc 56
+commons-lang bbd990b81fd7f8ab9dde75c8070b973b9ce500fc 57
+commons-lang 1fd45a4f68ea497dfbdf1a67b8b8805a4a9b1166 58
+commons-lang c04cd2337786ce7d54ed1fb757192fb8bc8c5e41 59
+commons-lang a8203b65261110c4a30ff69fe0da7a2390d82757 60
+commons-lang 88c76d207f642831ca899e144159424aaa60cf8e 61
+commons-lang 1313a65c11dea54f1baf492d4185fb2d673b6716 62
+commons-lang 0695c3f71e3fc5abe29d8b9b33eed7faac7e73bf 63
+commons-lang 8de71ae52958ae70fe651b7327319c9beecd9b84 64
+commons-lang bb1671667d9085fb76bd06af7f7ad91d2c1c8389 65
+closure-compiler 1dfad5043a207e032a78ef50c3cba50488bcd300 1
+closure-compiler d1cfe67977d8f3aaa85ec20c262171da394d5977 2
+closure-compiler d80fcc04239ab8c4cf781273c4f9bc54cf06f479 3
+closure-compiler efefb736fccc2039b5fb079710b3f2ac82b8c6e4 4
+closure-compiler 59eec92e364b2ec2cec9dd63449f5c0134983f18 5
+closure-compiler 9f7a353385bf0b93cdaeaa56cc5b1450db790127 6
+closure-compiler e736b1c92981fafdc87f3f41f66918305f173734 7
+closure-compiler 6300a43566d4adcf8828128267808a8740edc835 8
+closure-compiler e9c15465c000c19ef4e0b8f68a680589ae4111d7 9
+closure-compiler 0884a4cbef1c82153ef306477a12af0480385a35 10
+closure-compiler 482b767639e6eec7c30d8898851560c41b66d885 11
+closure-compiler d06ac163013a3779fcec7c2f544ba99d08f69f58 12
+closure-compiler d6d733f461ee5ba26e9326c1c7609f1f00d6d2db 13
+closure-compiler 4b15b25f400335b6e2820cb690430324748372f9 14
+closure-compiler 968b3f467f51c6bfec18af1fbcf980a0f19a1fb3 15
+closure-compiler 3b97e5e4212c769daa22a96094ed19b4658c8760 16
+closure-compiler 33ab4df84afbdb36216b8a1ea3a526c438615e26 17
+closure-compiler ea119a2d06328852763f9ee447c3bf74fa1f83dc 18
+closure-compiler 686146c4de7d2bd985156ae15175365633dd0f94 19
+closure-compiler ebf229b05f4ee71bc05d70830f5dd5683271d661 20
+closure-compiler 1787d7b2cff83a0f177dd958033c53e4502406d2 21
+closure-compiler 43a55234ef122a1ed98681ce0350506207b878d5 22
+closure-compiler b59cdd9e2dc64d2db86b90982c200a78f2f85e50 23
+closure-compiler 70ca6014779735bdc52ca0e80e528257829a15e7 24
+closure-compiler 2e904fcdc3bc09e3bff557fcfca383ba7e450095 25
+closure-compiler 372eb11e44b8559004bfe78a5b4003caf0072135 26
+closure-compiler 7a8983042131c5a9c2b5421ae42075cbb8ef1aeb 27
+closure-compiler 59dbb7a28680caad08ddb1658123fe0cbb1ae689 28
+closure-compiler 6924915efab89f921fd0779fe200fecc1bf80847 29
+closure-compiler 38e2b8f247423734adec3dbd76fa22da1a3561a8 30
+closure-compiler 0a670cb51fd56c12c19f98884c9513792ebc3f76 31
+closure-compiler 66c8c695fd8130e3ede6f77c6e5cc1ec96a0e9b3 32
+closure-compiler 382422adae8e9f07fc23c94089c0ebe08a2174bc 33
+closure-compiler 87b154f89cba3c67045cd7a783b79231d0630a4c 34
+closure-compiler 22784dc96c391b01692ce686eb93b9aa0ef74ede 35
+closure-compiler be397ee0d0be2aad8f71d12f097b17563866445a 36
+closure-compiler 7e93d521cb67ce33018315234fafe8a5aa30ee1e 37
+closure-compiler 8cf08115c6b252f8c6b54348a4089dccc45a01fb 38
+closure-compiler d577973fb378dfe0d05991ec0dfa6c1f40e574ee 39
+closure-compiler 6ebc0c3d7d4da7d2cf95f53d4ea790e89d3abc7a 40
+closure-compiler 00b1517c53912ac6843c257c0a6ead1176e8c5ee 41
+closure-compiler fb01427a445ebfeb3af5e220a98ccaced339b73e 42
+closure-compiler f636f23bdef4b27752ca249a28463ec1b6af1f0b 43
+closure-compiler 64bb2291f9a9bbab67d865dffe603f8a0df8ef30 44
+closure-compiler 6aa71f116df0102a98c463f471ad8ddd4dba01f8 45
+closure-compiler 4a1a31c6a50a0fbe25fa33277909bd51f1deb8e9 46
+closure-compiler aedce35970673f20696d5721acba13e986cc764c 47
+closure-compiler 46da17d59abb4f9c48b6ffd31601c212490b970b 48
+closure-compiler 1a7ba5cb0111dd3ed9afd9691cab39b3c341e408 49
+closure-compiler c3b630fc9c2a1c4eb7cb718f8d324bfb306cb9df 50
+closure-compiler a02241e5df48e44e23dc0e66dbef3fdc3c91eb3e 51
+closure-compiler 5909c3db59b16886917456ab950ef9661a7fe6b1 52
+closure-compiler 9959716b01fc5231ae68bc7a24454ce45d341606 53
+closure-compiler 7e9b8336568ea7f05c33e1b58eb67fa6d4eee756 54
+closure-compiler f2c0cb1f67ecb3315c0cb11326c9cb04db07ee8a 55
+closure-compiler 9e921fa973c603473f745fff76c44620fd7719e3 56
+closure-compiler f759f431d5a7a2261b44440bca9cfab93ae25302 57
+closure-compiler f640b5dec722422c93a9a624cd21d30b5e321858 58
+closure-compiler 49b4ef31c850611a5c85d95140b4cedda7f4c59a 59
+closure-compiler fbaadc736f0031dd84bebea33abae497b47158cb 60
+closure-compiler 34e4616a924bf272c076039e143e20f1d792731e 61
+closure-compiler b12d1d6489329c989b15635f6f7f06681b3f6582 62
+closure-compiler 37b0e1b0b5860e6cd4f48519c1105623778dc44f 63
+closure-compiler de726510af4329fb8eeb8a54b2d93d8d37c87545 64
+closure-compiler 48aea0ff7d027507362007415963a4b662fb10a7 65
+closure-compiler 59a30b48325cccadabae8687fdf603d72e26c157 66
+closure-compiler aa89d05b48332d68b8e6ebcee427b2ac3b4e3184 67
+closure-compiler 9d5a6e3082dd99f6c44df1b3526b9e83b79aa7da 68
+closure-compiler 63df133d0850805818d1e280ba47dcefc9763c89 69
+closure-compiler ec4a34c2bd87ba54fa8770affdeeea4f3c42089b 70
+closure-compiler db1fdf97454a6e4ff2d29cab5f1ef2273b54c0b4 71
+closure-compiler e323b879de2e53e5eedf6e7afb0d582382c6248d 72
+closure-compiler ee8344091722c28c9351e78dcb4155fbfddfd1af 73
+closure-compiler 03fee32a06ccf8257b2da67b9034270be38805ae 74
+closure-compiler c6cc58a9bc617483366141386e047b085b4e83ff 75
+closure-compiler 78b01c3a435cd175ce3ee70f97b2f8faac342cdc 76
+closure-compiler b88400ce0e9c1a0dadfd3417c256b02a82f38fe1 77
+closure-compiler 25829b0395164533782d608399096803321225a7 78
+closure-compiler 5d397618f3c86d9c444a4c4c6441267b8a89a21d 79
+closure-compiler 58786c3f717fa506280127265cd68fedf17de0a9 80
+closure-compiler 084b868bc2fbf9d6d0319e5a362ade1d7f43f899 81
+closure-compiler a857aec27d79fc234fddd4a72c1215af0251b2d8 82
+closure-compiler 43c245f0ff8d409e81e25687e69d34666b7cf26a 83
+closure-compiler 4839e1ea57f9a1a26debd2cc8f22beef7282b490 84
+closure-compiler 21a2103d7fa5664ea324ef9ee25b4a8922e50955 85
+closure-compiler 0907b6618a60b2de23c8f7ec2217a37dc5e9a091 86
+closure-compiler f64a3f51bd346547ce6e1edb30601df73dff27d9 87
+closure-compiler 6a17da87f9e132bdc147b33223826da040c77a6b 88
+closure-compiler 4b065734d8afb5ab0d241ee5da22af0fa9d75ec3 89
+closure-compiler c5e143f7df0c3d73f6634488cac8ad8e7054fe05 90
+closure-compiler 7eb2d84de101a125d41d3a1157f4eed789ca4a0d 91
+closure-compiler 8980b60033b1c6853cbca593ec89d3d332adf7dc 92
+closure-compiler 9d1e18e87a41f2a610ed70f3a1d57abbce7a1cd7 93
+closure-compiler 7cc748592670105f9d783b1b85f0f27e938db4ff 94
+closure-compiler f6607996d6d3bd6820ce3848d147991b27600a12 95
+closure-compiler 807aaf2569e9eb09761068c5aeb95654daea30d9 96
+closure-compiler ee749e286b477f8d6f53ff5960a38453baf50f31 97
+closure-compiler d1df970451b5a18956448097b3afb43f3a82263d 98
+closure-compiler 8aa879a4eeb18cfc8d13e6c843a32b7f41ccd516 99
+closure-compiler 6d00ac1a68612aade8a19d7ecc9b180f00ae5234 100
+closure-compiler 369282db85567f1bf2c9635d0c0043d47f114814 101
+closure-compiler d48c365ba8251057a71a2e2b7aabff640209e31b 102
+closure-compiler 2f5cb1622371de540fe20dcbe0411651ec89f952 103
+closure-compiler 67289ae4cbaba3ae70cd2e8fb92f3f2898039dfb 104
+closure-compiler 113a08f59e8116fb11a7d4ceb11d5bb09b74ac3c 105
+closure-compiler e609670bb56e0c7216c9476a5f6a5594e7f1acdb 106
+closure-compiler 49e9565febba904484396e2aef7dbe86f55e9cc5 107
+closure-compiler aae547f4d911dfbf42ca5a31aeb401ac83725a14 108
+closure-compiler 7d26b7bcae8311f3c032bae3f1df27b590a4c174 109
+closure-compiler 13fb7fcbcad87f8df63535491627bf7b01471064 110
+closure-compiler 6f981f77e5bd9e2eecbd851a2a02933cdb9af0d8 111
+closure-compiler b168c8822b2d3d12515ec6477812674c8805bcf3 112
+closure-compiler 0fb76a81bbdd8ab84a00b8be2099abfb83c88668 113
+closure-compiler 5f03c6817341d39d8dead76c7511fd6bb0b9009f 114
+closure-compiler 4597738e8898f738c1f969fe90479728be81cc80 115
+closure-compiler b80dfd3b0e5e4b490b7afb469320d18bac4520b4 116
+closure-compiler 871bca1f36c5c06aa880ce21429da63a76f8f6e9 117
+closure-compiler f17961ae4d3d7cf5ea291d82804239982d009ef7 118
+closure-compiler 046e0b226e963bc584c2a6cd3c2b471777c5cc53 119
+closure-compiler 0123fd5303c85d0d26add64aa2e19fee33f73aaa 120
+closure-compiler 2aee36e667526ff8b0b5e6dad66506acee920ea6 121
+closure-compiler a8a456b183c9600658bcb0c72091bae1a30a4fda 122
+closure-compiler 8d11b8cc7a6183222b61c9bd398fed5448c59109 123
+closure-compiler e7f4269d0289f4d47217207ec456219db8efe47c 124
+closure-compiler 8cef00180a5cf67d047919c89668a6405030dbab 125
+closure-compiler bd2803b6d9ab600906b262ae51cb3591160b5f3c 126
+closure-compiler 211dc0161ae737f59cac22f30b048d56a059d14b 127
+closure-compiler d82fb38b0121bf690cea58df293185c7e91ded9c 128
+closure-compiler 79a8906570f44e5f519282bd32595985ed089aed 129
+closure-compiler 9fac3ccd2fa9e6137584f079db1a6f5962a65cf4 130
+closure-compiler 7f6700e2b54af3af409f3e8851a0d98a72beef4b 131
+closure-compiler 5b9485903b5e7d926f49dc91b915a256df92591c 132
+closure-compiler 4fbbc47cb18f241b23dd2d4bf9c15d45b2473523 133
+joda-time 9a62b06be5d0df8e833ff8583398cca386608cac 1
+joda-time 8612f9e5b88c1bea933ef9ab1e431f5db3006b48 2
+joda-time 8d109fe1a999a11b4557536dd96f9210460a5936 3
+joda-time 3ba9ba799b3261b7332a467a88be142c83b298fd 4
+joda-time a6cb59ed2280ab0a32995fa8b5f1a7b0d47cb815 5
+joda-time 28aeba952e838fb12c9b934ce6ef65658e45d7b2 6
+joda-time 1adb1e69863dcd1ff282692bf1452c422528eeb9 7
+joda-time 362ed1787724cda3ded37de2b0aa0f13adf0a66e 8
+joda-time 96f586cb8dac7eee62e9be519a3eab125c47bb23 9
+joda-time 3a413d7844c22dc6ddd50bf5d0d55ff3589e47ac 10
+joda-time 57eb4cbb9044771cd46a9eee0c62016618930226 11
+joda-time 77a6b3a44da7ae4af62e88b4b27eeefec2d768c2 12
+joda-time 5d08a1cdbfa6ce86a0baf637493c464e91e91968 13
+joda-time 2ea856328ed5710537778d755a5ff52a6831a2ee 14
+joda-time 0cefc4c212e92e5dccdfa658785c419483317558 15
+joda-time 706513d59425e7a9dc6bdb972f25b03b02e48558 16
+joda-time a8913f5ad5dfde59263c981c9e9eb28e43fd2e00 17
+joda-time b609d7d66d180f2eb537b27f5d605d0596e096ce 18
+joda-time 82c5e4f9550e4df36e07b66f35f7c3e9d7eb5eba 19
+joda-time a92450e88df85d6b7a0fa53517da46286c24f53f 20
+joda-time 0973b46039c4db0be3a969d36c220dee1e1fb4da 21
+joda-time 57c8aaf94b95323c295dcfec5f40f181846164e7 22
+joda-time 14dedcbc04682c1b1b6c5ebe91bc930b79eeb572 23
+joda-time d50efdf75a8daeced88d79b168f68d189fb87e13 24
+joda-time 552be4b677ec30a34d04d234395ba1a8c7beaacf 25
+joda-time c7a581e55fc988bd90fa4bb1b0acece5181b7c5f 26
+joda-time e0559c503f65641b9546c37e7c84c866caf37e66 27
+mockito 4e9d7607825c3c668fd43f19507bfead566c528c 1
+mockito 80452c7a42777513c35fd30c4e12bcd7ee438fb9 2
+mockito 3eec7451d6c83c280743c39b39c77a179abb30f9 3
+mockito 42b391d4209024292b6101048389570e4ec36b2a 4
+mockito 42a24dde02923185db3f79ae57e7819f7d70af55 5
+mockito dc205824dbc289acbcde919e430176ad72da847f 6
+mockito 0aaa36ce6924ca87f88e3723771413b42b80dc5a 7
+mockito 5a03bf5d0c9aedac9cfbf074833167c1eca6439f 8
+mockito 31f908029b8cd9b1f32bed3a41651b30ebb47b9f 9
+mockito 1beeae39ac9f79d6a0db285dec311b78025ac062 10
+mockito 1802cf79c482f2df29abb3d6735bcf0b16cedb7d 11
+mockito 7a647a702c8af81ccf5d37b09c11529c6c0cb1b7 12
+mockito 2037361bea014c9ac87393eb96a95374dff46182 13
+mockito 9b2cf8072ece7c7629eff6037853b3e14ab5f524 14
+mockito 81a380951402d2a158af115ed616ab36e94793b8 15
+mockito 620f8faed0ad132961aae3a8671120e0929ce828 16
+mockito 77cb2037314dd024eb53ffe2e9e06304088a2d53 17
+mockito 8ceb04ad7317d8db24476b3b5f56ec9f658bc43a 18
+mockito 110ffa80070bf54ab8efabdaaa27b193a93d6128 19
+mockito 7616cd17d5e1dc7c3dee080c6969183821d4797a 20
+mockito 8912aa176ea8414c2fc57df0d9b030b918630e9f 21
+mockito d0c872e4b0837aef1e1635bf5f15d33c3d8d9698 22
+mockito 82935114a09390cbab0c6b6df9b6fd6788bf55d9 23
+mockito c17169c2cbb0b3d055d64ec2c4859122ca919c42 24
+mockito a6ccf070a267e0a165b8efaaefa8938e787bb6a0 25
+mockito af44738c7de74f24e37ea0c1242e73b07c3f4362 26
+mockito a8ec4fa290bc1cdde060218d6bb811ac1fa6da6f 27
+mockito 0464f5657dc8c76a4ff209a2cf7fd6312885cdc3 28
+mockito 918f0a5aed6454b307004b6c9c86afc8e96353ff 29
+mockito 3c924f80a4db3692a13341f0da4517052ab77a2a 30
+mockito 62b6bdf44baeee172b2b1684835fb995bad2a47d 31
+mockito 4f7060cc54bb50292ccfbf3b9fca27f0282abf51 32
+mockito 5cb37751c0aba81d025dc5fe4f2f9a3bf86cc958 33
+mockito e8cebe01abae49a09aadd6a26c6f7e2f90e5e772 34
+mockito 27a2f515197d4688c4df82afb791bf8ce20a5504 35
+mockito 84c142f56fe971ed52ecfe42b42bc0aa7bef8354 36
+mockito c0222c2dbfbd2b053961a46e4c2a3973aec55a75 37
+mockito c1f2c4e6b5bab4cfc004f22999e11e8ac140d377 38

python/data/introClassData.txt

@@ -0,0 +1,765 @@
+introclass:checksum:08c7ea:006
+introclass:checksum:08c7ea:007
+introclass:checksum:08c7ea:010
+introclass:checksum:08c7ea:011
+introclass:checksum:08c7ea:012
+introclass:checksum:08c7ea:013
+introclass:checksum:30074a:007
+introclass:checksum:3214e9:000
+introclass:checksum:3214e9:001
+introclass:checksum:3214e9:003
+introclass:checksum:3214e9:004
+introclass:checksum:3b2376:006
+introclass:checksum:659a73:002
+introclass:checksum:659a73:003
+introclass:checksum:98d873:000
+introclass:checksum:98d873:001
+introclass:checksum:98d873:002
+introclass:checksum:a0e3fd:000
+introclass:checksum:a0e3fd:001
+introclass:checksum:a0e3fd:002
+introclass:checksum:a0e3fd:003
+introclass:checksum:a0e3fd:004
+introclass:checksum:a0e3fd:005
+introclass:checksum:b6fd40:000
+introclass:checksum:b6fd40:001
+introclass:checksum:ca94e3:006
+introclass:checksum:d43d32:000
+introclass:checksum:d5cc7a:000
+introclass:checksum:d5cc7a:001
+introclass:checksum:d5cc7a:002
+introclass:digits:070455:000
+introclass:digits:070455:001
+introclass:digits:08c7ea:000
+introclass:digits:08c7ea:001
+introclass:digits:0cdfa3:000
+introclass:digits:0cdfa3:001
+introclass:digits:0cdfa3:002
+introclass:digits:0cdfa3:003
+introclass:digits:0cea42:000
+introclass:digits:0cea42:001
+introclass:digits:0cea42:002
+introclass:digits:1391c9:000
+introclass:digits:1391c9:001
+introclass:digits:1391c9:002
+introclass:digits:1c2bb3:000
+introclass:digits:1c2bb3:001
+introclass:digits:1c2bb3:002
+introclass:digits:1c2bb3:003
+introclass:digits:1c2bb3:005
+introclass:digits:1c2bb3:006
+introclass:digits:295afd:000
+introclass:digits:295afd:001
+introclass:digits:295afd:003
+introclass:digits:2af3c4:000
+introclass:digits:2af3c4:001
+introclass:digits:313d57:000
+introclass:digits:317aa7:001
+introclass:digits:317aa7:002
+introclass:digits:317aa7:004
+introclass:digits:3214e9:000
+introclass:digits:387be3:002
+introclass:digits:65e02c:004
+introclass:digits:65e02c:008
+introclass:digits:6e464f:003
+introclass:digits:6e464f:004
+introclass:digits:88394f:000
+introclass:digits:88394f:001
+introclass:digits:88394f:002
+introclass:digits:88394f:003
+introclass:digits:88394f:004
+introclass:digits:8cddfd:005
+introclass:digits:90a14c:004
+introclass:digits:98d873:000
+introclass:digits:98d873:001
+introclass:digits:98d873:002
+introclass:digits:98d873:003
+introclass:digits:a0e3fd:003
+introclass:digits:a3d177:000
+introclass:digits:c9d718:001
+introclass:digits:ca5057:000
+introclass:digits:ca5057:001
+introclass:digits:ca5057:002
+introclass:digits:ca5057:003
+introclass:digits:ca94e3:000
+introclass:digits:ca94e3:001
+introclass:digits:ca94e3:002
+introclass:digits:cd2d9b:003
+introclass:digits:cd2d9b:004
+introclass:digits:d25c71:000
+introclass:digits:d25c71:001
+introclass:digits:d25c71:002
+introclass:digits:d25c71:003
+introclass:digits:d25c71:004
+introclass:digits:d25c71:005
+introclass:digits:d2b889:006
+introclass:digits:d43d32:000
+introclass:digits:d5059e:000
+introclass:digits:d6364e:000
+introclass:digits:d6364e:001
+introclass:digits:d767ad:000
+introclass:digits:d767ad:001
+introclass:digits:d767ad:002
+introclass:digits:d767ad:003
+introclass:digits:d767ad:004
+introclass:digits:d767ad:005
+introclass:digits:d767ad:006
+introclass:digits:d767ad:007
+introclass:digits:d8b262:000
+introclass:digits:f227ed:000
+introclass:digits:f2997e:000
+introclass:digits:f2997e:001
+introclass:digits:f2997e:002
+introclass:digits:f5b56c:000
+introclass:grade:070455:001
+introclass:grade:070455:002
+introclass:grade:0cdfa3:003
+introclass:grade:0cea42:000
+introclass:grade:0cea42:001
+introclass:grade:0cea42:002
+introclass:grade:0cea42:003
+introclass:grade:0cea42:004
+introclass:grade:1b31fa:000
+introclass:grade:1b31fa:001
+introclass:grade:1b31fa:002
+introclass:grade:1b31fa:003
+introclass:grade:1c2bb3:000
+introclass:grade:1c2bb3:001
+introclass:grade:1c2bb3:002
+introclass:grade:1c2bb3:003
+introclass:grade:295afd:000
+introclass:grade:295afd:001
+introclass:grade:295afd:002
+introclass:grade:295afd:003
+introclass:grade:295afd:004
+introclass:grade:295afd:005
+introclass:grade:295afd:006
+introclass:grade:295afd:007
+introclass:grade:295afd:008
+introclass:grade:295afd:009
+introclass:grade:295afd:010
+introclass:grade:295afd:011
+introclass:grade:295afd:012
+introclass:grade:30074a:001
+introclass:grade:30074a:002
+introclass:grade:30074a:003
+introclass:grade:30074a:004
+introclass:grade:317aa7:000
+introclass:grade:317aa7:001
+introclass:grade:317aa7:002
+introclass:grade:317aa7:003
+introclass:grade:317aa7:004
+introclass:grade:36d800:000
+introclass:grade:387be3:000
+introclass:grade:3b2376:000
+introclass:grade:3b2376:001
+introclass:grade:3b2376:002
+introclass:grade:3b2376:003
+introclass:grade:3b2376:004
+introclass:grade:3b2376:005
+introclass:grade:3b2376:006
+introclass:grade:3b2376:007
+introclass:grade:3cf6d3:001
+introclass:grade:3cf6d3:002
+introclass:grade:3cf6d3:003
+introclass:grade:3cf6d3:004
+introclass:grade:3cf6d3:005
+introclass:grade:3cf6d3:006
+introclass:grade:3cf6d3:007
+introclass:grade:3cf6d3:008
+introclass:grade:3cf6d3:009
+introclass:grade:3cf6d3:010
+introclass:grade:3cf6d3:011
+introclass:grade:3cf6d3:012
+introclass:grade:3cf6d3:013
+introclass:grade:489253:000
+introclass:grade:489253:001
+introclass:grade:489253:002
+introclass:grade:489253:003
+introclass:grade:489253:004
+introclass:grade:489253:005
+introclass:grade:489253:006
+introclass:grade:489253:007
+introclass:grade:489253:008
+introclass:grade:489253:009
+introclass:grade:489253:010
+introclass:grade:489253:011
+introclass:grade:48b829:000
+introclass:grade:531924:000
+introclass:grade:531924:001
+introclass:grade:5b504b:000
+introclass:grade:68ea5d:000
+introclass:grade:6e464f:000
+introclass:grade:75c98d:001
+introclass:grade:75c98d:002
+introclass:grade:75c98d:003
+introclass:grade:75c98d:004
+introclass:grade:769cd8:000
+introclass:grade:833bd4:000
+introclass:grade:833bd4:001
+introclass:grade:833bd4:002
+introclass:grade:833bd4:003
+introclass:grade:89b1a7:000
+introclass:grade:89b1a7:001
+introclass:grade:89b1a7:002
+introclass:grade:89b1a7:003
+introclass:grade:89b1a7:004
+introclass:grade:89b1a7:005
+introclass:grade:8ce634:000
+introclass:grade:8ce634:001
+introclass:grade:9013bd:000
+introclass:grade:908348:009
+introclass:grade:908348:010
+introclass:grade:908348:011
+introclass:grade:908348:013
+introclass:grade:92b7dd:002
+introclass:grade:93f87b:013
+introclass:grade:93f87b:014
+introclass:grade:93f87b:015
+introclass:grade:93f87b:016
+introclass:grade:953627:006
+introclass:grade:953627:007
+introclass:grade:953627:008
+introclass:grade:953627:009
+introclass:grade:953627:010
+introclass:grade:953627:011
+introclass:grade:953627:012
+introclass:grade:953627:013
+introclass:grade:953627:014
+introclass:grade:953627:015
+introclass:grade:953627:016
+introclass:grade:953627:017
+introclass:grade:953627:018
+introclass:grade:97f6b1:000
+introclass:grade:98d873:004
+introclass:grade:9c9308:000
+introclass:grade:9c9308:001
+introclass:grade:9c9308:002
+introclass:grade:9c9308:003
+introclass:grade:9c9308:004
+introclass:grade:9c9308:005
+introclass:grade:9c9308:006
+introclass:grade:9c9308:007
+introclass:grade:a0e3fd:000
+introclass:grade:a0e3fd:001
+introclass:grade:a0e3fd:002
+introclass:grade:a0e3fd:003
+introclass:grade:a0e3fd:004
+introclass:grade:a0e3fd:005
+introclass:grade:a0e3fd:009
+introclass:grade:a0e3fd:010
+introclass:grade:a0e3fd:011
+introclass:grade:a0e3fd:012
+introclass:grade:aaceaf:000
+introclass:grade:aaceaf:001
+introclass:grade:aaceaf:002
+introclass:grade:aaceaf:003
+introclass:grade:aaceaf:004
+introclass:grade:aaceaf:005
+introclass:grade:aaceaf:006
+introclass:grade:af81ff:000
+introclass:grade:af81ff:001
+introclass:grade:af81ff:002
+introclass:grade:af81ff:003
+introclass:grade:af81ff:004
+introclass:grade:af81ff:005
+introclass:grade:af81ff:006
+introclass:grade:af81ff:007
+introclass:grade:af81ff:008
+introclass:grade:af81ff:009
+introclass:grade:b1924d:000
+introclass:grade:b1924d:001
+introclass:grade:b1924d:002
+introclass:grade:b1924d:003
+introclass:grade:b6fd40:000
+introclass:grade:bfad6d:000
+introclass:grade:bfad6d:001
+introclass:grade:c9d718:000
+introclass:grade:c9d718:001
+introclass:grade:c9d718:002
+introclass:grade:c9d718:003
+introclass:grade:c9d718:004
+introclass:grade:ca94e3:000
+introclass:grade:cb243b:000
+introclass:grade:cb243b:001
+introclass:grade:cd2d9b:001
+introclass:grade:cd2d9b:002
+introclass:grade:cd2d9b:003
+introclass:grade:cd2d9b:004
+introclass:grade:cd2d9b:005
+introclass:grade:cd2d9b:006
+introclass:grade:cd2d9b:007
+introclass:grade:cd2d9b:008
+introclass:grade:cd2d9b:009
+introclass:grade:cd2d9b:010
+introclass:grade:cd2d9b:011
+introclass:grade:cd2d9b:012
+introclass:grade:cd2d9b:013
+introclass:grade:d009aa:000
+introclass:grade:d009aa:001
+introclass:grade:d009aa:002
+introclass:grade:d009aa:003
+introclass:grade:d009aa:004
+introclass:grade:d009aa:005
+introclass:grade:d6364e:000
+introclass:grade:d6364e:001
+introclass:grade:d6364e:002
+introclass:grade:d6364e:003
+introclass:grade:d6364e:004
+introclass:grade:d6364e:005
+introclass:grade:d6364e:006
+introclass:grade:d6364e:007
+introclass:grade:d8b262:000
+introclass:grade:d8b262:001
+introclass:grade:dc11fb:000
+introclass:grade:dccb12:001
+introclass:grade:e23b96:000
+introclass:grade:e23b96:001
+introclass:grade:e23b96:002
+introclass:grade:e23b96:003
+introclass:grade:e23b96:004
+introclass:grade:e23b96:005
+introclass:grade:ee1f20:000
+introclass:grade:ee1f20:001
+introclass:grade:f5b56c:000
+introclass:grade:f5b56c:001
+introclass:grade:f5b56c:002
+introclass:grade:f5b56c:003
+introclass:grade:f5b56c:004
+introclass:grade:f5b56c:005
+introclass:grade:f5b56c:006
+introclass:grade:f5b56c:007
+introclass:grade:f5b56c:008
+introclass:grade:f5b56c:009
+introclass:grade:f5b56c:010
+introclass:grade:f5b56c:011
+introclass:grade:f5b56c:012
+introclass:grade:fe9d5f:000
+introclass:grade:fe9d5f:001
+introclass:grade:fe9d5f:002
+introclass:grade:fe9d5f:003
+introclass:grade:fe9d5f:004
+introclass:median:070455:000
+introclass:median:070455:001
+introclass:median:0cdfa3:002
+introclass:median:0cdfa3:003
+introclass:median:0cdfa3:004
+introclass:median:0cea42:000
+introclass:median:0cea42:001
+introclass:median:0cea42:002
+introclass:median:0cea42:003
+introclass:median:0cea42:004
+introclass:median:0cea42:005
+introclass:median:0cea42:006
+introclass:median:0cea42:007
+introclass:median:0cea42:008
+introclass:median:15cb07:001
+introclass:median:15cb07:002
+introclass:median:15cb07:003
+introclass:median:15cb07:004
+introclass:median:1b31fa:000
+introclass:median:1b31fa:001
+introclass:median:1b31fa:002
+introclass:median:1c2bb3:000
+introclass:median:1c2bb3:001
+introclass:median:1c2bb3:002
+introclass:median:2c1556:000
+introclass:median:30074a:000
+introclass:median:30074a:001
+introclass:median:30074a:002
+introclass:median:317aa7:000
+introclass:median:317aa7:001
+introclass:median:317aa7:002
+introclass:median:317aa7:003
+introclass:median:3b2376:000
+introclass:median:3b2376:001
+introclass:median:3b2376:002
+introclass:median:3b2376:003
+introclass:median:3b2376:004
+introclass:median:3b2376:005
+introclass:median:3b2376:006
+introclass:median:3cf6d3:000
+introclass:median:3cf6d3:001
+introclass:median:3cf6d3:002
+introclass:median:3cf6d3:003
+introclass:median:3cf6d3:004
+introclass:median:3cf6d3:005
+introclass:median:3cf6d3:006
+introclass:median:3cf6d3:007
+introclass:median:489253:007
+introclass:median:593b95:000
+introclass:median:593b95:001
+introclass:median:593b95:002
+introclass:median:593b95:003
+introclass:median:593b95:004
+introclass:median:593b95:005
+introclass:median:593b95:006
+introclass:median:593b95:007
+introclass:median:68eb0b:000
+introclass:median:89b1a7:002
+introclass:median:89b1a7:003
+introclass:median:89b1a7:005
+introclass:median:89b1a7:006
+introclass:median:89b1a7:007
+introclass:median:89b1a7:008
+introclass:median:89b1a7:009
+introclass:median:89b1a7:010
+introclass:median:8c7dfb:000
+introclass:median:8c7dfb:001
+introclass:median:8c7dfb:002
+introclass:median:8c7dfb:003
+introclass:median:8c7dfb:004
+introclass:median:8c7dfb:005
+introclass:median:8c7dfb:006
+introclass:median:8c7dfb:007
+introclass:median:8c7dfb:008
+introclass:median:9013bd:000
+introclass:median:908348:000
+introclass:median:908348:001
+introclass:median:908348:002
+introclass:median:908348:003
+introclass:median:908348:004
+introclass:median:908348:005
+introclass:median:908348:006
+introclass:median:908348:007
+introclass:median:908348:008
+introclass:median:908348:009
+introclass:median:908348:010
+introclass:median:908348:011
+introclass:median:908348:012
+introclass:median:908348:013
+introclass:median:908348:014
+introclass:median:908348:015
+introclass:median:90a14c:000
+introclass:median:90a14c:001
+introclass:median:90a14c:002
+introclass:median:93f87b:001
+introclass:median:93f87b:002
+introclass:median:93f87b:003
+introclass:median:93f87b:004
+introclass:median:93f87b:005
+introclass:median:93f87b:006
+introclass:median:93f87b:007
+introclass:median:93f87b:008
+introclass:median:93f87b:009
+introclass:median:93f87b:010
+introclass:median:93f87b:011
+introclass:median:93f87b:012
+introclass:median:93f87b:013
+introclass:median:93f87b:014
+introclass:median:93f87b:015
+introclass:median:93f87b:016
+introclass:median:953627:000
+introclass:median:953627:001
+introclass:median:953627:002
+introclass:median:953627:003
+introclass:median:97f6b1:000
+introclass:median:97f6b1:001
+introclass:median:97f6b1:003
+introclass:median:98d873:000
+introclass:median:98d873:001
+introclass:median:9c9308:000
+introclass:median:9c9308:001
+introclass:median:9c9308:002
+introclass:median:9c9308:003
+introclass:median:9c9308:004
+introclass:median:9c9308:006
+introclass:median:9c9308:007
+introclass:median:9c9308:008
+introclass:median:9c9308:009
+introclass:median:9c9308:012
+introclass:median:aaceaf:000
+introclass:median:aaceaf:001
+introclass:median:aaceaf:002
+introclass:median:aaceaf:003
+introclass:median:aaceaf:004
+introclass:median:aaceaf:005
+introclass:median:aaceaf:006
+introclass:median:af81ff:000
+introclass:median:af81ff:001
+introclass:median:af81ff:002
+introclass:median:af81ff:003
+introclass:median:af81ff:004
+introclass:median:af81ff:006
+introclass:median:af81ff:007
+introclass:median:b6fd40:000
+introclass:median:b6fd40:001
+introclass:median:c716ee:000
+introclass:median:c716ee:001
+introclass:median:c716ee:002
+introclass:median:d009aa:000
+introclass:median:d009aa:001
+introclass:median:d009aa:002
+introclass:median:d12048:000
+introclass:median:d2b889:000
+introclass:median:d4aae1:000
+introclass:median:d6364e:005
+introclass:median:d6364e:006
+introclass:median:e9c620:000
+introclass:median:e9c620:001
+introclass:median:fcf701:000
+introclass:median:fcf701:001
+introclass:median:fcf701:002
+introclass:median:fe9d5f:000
+introclass:median:fe9d5f:001
+introclass:median:fe9d5f:002
+introclass:smallest:070455:000
+introclass:smallest:070455:001
+introclass:smallest:070455:002
+introclass:smallest:1391c9:002
+introclass:smallest:1391c9:003
+introclass:smallest:1391c9:004
+introclass:smallest:1391c9:005
+introclass:smallest:15cb07:002
+introclass:smallest:15cb07:003
+introclass:smallest:15cb07:004
+introclass:smallest:15cb07:005
+introclass:smallest:15cb07:006
+introclass:smallest:15cb07:007
+introclass:smallest:1b31fa:003
+introclass:smallest:1b31fa:004
+introclass:smallest:1b31fa:005
+introclass:smallest:1b31fa:006
+introclass:smallest:2694af:000
+introclass:smallest:30074a:000
+introclass:smallest:30074a:001
+introclass:smallest:30074a:002
+introclass:smallest:30074a:003
+introclass:smallest:30074a:004
+introclass:smallest:30074a:005
+introclass:smallest:30074a:006
+introclass:smallest:30074a:007
+introclass:smallest:30074a:008
+introclass:smallest:30074a:009
+introclass:smallest:346b1d:002
+introclass:smallest:346b1d:003
+introclass:smallest:346b1d:004
+introclass:smallest:346b1d:005
+introclass:smallest:346b1d:006
+introclass:smallest:346b1d:007
+introclass:smallest:346b1d:008
+introclass:smallest:346b1d:009
+introclass:smallest:346b1d:010
+introclass:smallest:346b1d:011
+introclass:smallest:36d800:000
+introclass:smallest:36d800:001
+introclass:smallest:36d800:002
+introclass:smallest:36d800:003
+introclass:smallest:36d800:004
+introclass:smallest:36d800:005
+introclass:smallest:3b2376:000
+introclass:smallest:3b2376:001
+introclass:smallest:3b2376:002
+introclass:smallest:3b2376:003
+introclass:smallest:3b2376:004
+introclass:smallest:3b2376:008
+introclass:smallest:3cf6d3:003
+introclass:smallest:3cf6d3:004
+introclass:smallest:3cf6d3:005
+introclass:smallest:3cf6d3:006
+introclass:smallest:48b829:000
+introclass:smallest:48b829:001
+introclass:smallest:5a5683:000
+introclass:smallest:5a5683:001
+introclass:smallest:5a5683:002
+introclass:smallest:68eb0b:000
+introclass:smallest:68eb0b:001
+introclass:smallest:68eb0b:002
+introclass:smallest:6aaeaf:000
+introclass:smallest:6aaeaf:001
+introclass:smallest:75c98d:000
+introclass:smallest:769cd8:000
+introclass:smallest:769cd8:001
+introclass:smallest:769cd8:002
+introclass:smallest:769cd8:003
+introclass:smallest:769cd8:004
+introclass:smallest:769cd8:006
+introclass:smallest:769cd8:010
+introclass:smallest:818f8c:002
+introclass:smallest:818f8c:003
+introclass:smallest:846021:000
+introclass:smallest:846021:001
+introclass:smallest:846021:002
+introclass:smallest:846021:003
+introclass:smallest:846021:004
+introclass:smallest:846021:005
+introclass:smallest:846021:006
+introclass:smallest:846021:007
+introclass:smallest:846021:008
+introclass:smallest:846021:009
+introclass:smallest:846021:010
+introclass:smallest:846021:011
+introclass:smallest:88394f:006
+introclass:smallest:88394f:007
+introclass:smallest:88394f:008
+introclass:smallest:88394f:009
+introclass:smallest:8c7dfb:000
+introclass:smallest:8c7dfb:001
+introclass:smallest:8c7dfb:002
+introclass:smallest:8c7dfb:003
+introclass:smallest:8c7dfb:004
+introclass:smallest:8c7dfb:005
+introclass:smallest:8c7dfb:006
+introclass:smallest:8c7dfb:007
+introclass:smallest:8c7dfb:008
+introclass:smallest:8ce634:000
+introclass:smallest:8ce634:001
+introclass:smallest:9013bd:000
+introclass:smallest:9013bd:001
+introclass:smallest:908348:004
+introclass:smallest:908348:005
+introclass:smallest:90a14c:000
+introclass:smallest:90a14c:001
+introclass:smallest:93f87b:000
+introclass:smallest:93f87b:001
+introclass:smallest:953627:009
+introclass:smallest:97f6b1:000
+introclass:smallest:97f6b1:001
+introclass:smallest:97f6b1:002
+introclass:smallest:97f6b1:003
+introclass:smallest:98d873:000
+introclass:smallest:98d873:001
+introclass:smallest:98d873:002
+introclass:smallest:a0e3fd:000
+introclass:smallest:a0e3fd:001
+introclass:smallest:a0e3fd:002
+introclass:smallest:a0e3fd:003
+introclass:smallest:a0e3fd:004
+introclass:smallest:a0e3fd:005
+introclass:smallest:a0e3fd:006
+introclass:smallest:af81ff:000
+introclass:smallest:af81ff:001
+introclass:smallest:b1924d:000
+introclass:smallest:b1924d:001
+introclass:smallest:b1924d:002
+introclass:smallest:c868b3:000
+introclass:smallest:c868b3:001
+introclass:smallest:c868b3:002
+introclass:smallest:c9d718:000
+introclass:smallest:c9d718:001
+introclass:smallest:c9d718:002
+introclass:smallest:c9d718:003
+introclass:smallest:cb243b:000
+introclass:smallest:d009aa:001
+introclass:smallest:dedc2a:000
+introclass:smallest:e9c620:000
+introclass:smallest:ea67b8:000
+introclass:smallest:ea67b8:001
+introclass:smallest:ea67b8:002
+introclass:smallest:ea67b8:003
+introclass:smallest:ee1f20:006
+introclass:smallest:f2997e:000
+introclass:smallest:f2997e:001
+introclass:smallest:f2997e:002
+introclass:smallest:f8d57d:000
+introclass:smallest:f8d57d:001
+introclass:smallest:f8d57d:002
+introclass:smallest:fe9d5f:001
+introclass:syllables:0cea42:000
+introclass:syllables:0cea42:001
+introclass:syllables:0cea42:002
+introclass:syllables:0cea42:003
+introclass:syllables:0cea42:004
+introclass:syllables:0cea42:005
+introclass:syllables:0cea42:006
+introclass:syllables:2af3c4:000
+introclass:syllables:2af3c4:001
+introclass:syllables:2af3c4:002
+introclass:syllables:2af3c4:003
+introclass:syllables:36d800:000
+introclass:syllables:36d800:001
+introclass:syllables:36d800:002
+introclass:syllables:36d800:003
+introclass:syllables:36d800:004
+introclass:syllables:37593d:000
+introclass:syllables:37593d:001
+introclass:syllables:37593d:002
+introclass:syllables:37593d:003
+introclass:syllables:37593d:004
+introclass:syllables:37593d:005
+introclass:syllables:37593d:006
+introclass:syllables:38eb99:001
+introclass:syllables:38eb99:002
+introclass:syllables:38eb99:003
+introclass:syllables:38eb99:004
+introclass:syllables:489253:002
+introclass:syllables:489253:003
+introclass:syllables:489253:004
+introclass:syllables:489253:005
+introclass:syllables:489253:006
+introclass:syllables:489253:007
+introclass:syllables:593b95:000
+introclass:syllables:593b95:001
+introclass:syllables:593b95:002
+introclass:syllables:593b95:003
+introclass:syllables:68eb0b:000
+introclass:syllables:68eb0b:001
+introclass:syllables:68eb0b:002
+introclass:syllables:68eb0b:003
+introclass:syllables:68eb0b:004
+introclass:syllables:68eb0b:005
+introclass:syllables:68eb0b:006
+introclass:syllables:68eb0b:007
+introclass:syllables:68eb0b:008
+introclass:syllables:68eb0b:009
+introclass:syllables:818f8c:000
+introclass:syllables:818f8c:001
+introclass:syllables:818f8c:002
+introclass:syllables:818f8c:003
+introclass:syllables:818f8c:004
+introclass:syllables:818f8c:005
+introclass:syllables:818f8c:006
+introclass:syllables:818f8c:007
+introclass:syllables:88394f:000
+introclass:syllables:88394f:001
+introclass:syllables:88394f:002
+introclass:syllables:88394f:003
+introclass:syllables:88394f:004
+introclass:syllables:88394f:005
+introclass:syllables:9013bd:000
+introclass:syllables:90a14c:000
+introclass:syllables:90a14c:001
+introclass:syllables:90a14c:002
+introclass:syllables:93f87b:005
+introclass:syllables:97f6b1:000
+introclass:syllables:99cbb4:000
+introclass:syllables:99cbb4:001
+introclass:syllables:99cbb4:002
+introclass:syllables:99cbb4:003
+introclass:syllables:99cbb4:004
+introclass:syllables:99cbb4:005
+introclass:syllables:99cbb4:006
+introclass:syllables:b1924d:000
+introclass:syllables:b1924d:001
+introclass:syllables:b1924d:002
+introclass:syllables:b1924d:003
+introclass:syllables:b1924d:004
+introclass:syllables:b1924d:005
+introclass:syllables:b6fd40:000
+introclass:syllables:bfad6d:001
+introclass:syllables:ca5057:000
+introclass:syllables:ca5057:001
+introclass:syllables:ca5057:002
+introclass:syllables:ca5057:003
+introclass:syllables:ca5057:004
+introclass:syllables:cd2d9b:001
+introclass:syllables:cd2d9b:002
+introclass:syllables:cd2d9b:003
+introclass:syllables:cd2d9b:004
+introclass:syllables:d12048:000
+introclass:syllables:d5059e:000
+introclass:syllables:d5059e:001
+introclass:syllables:d5059e:002
+introclass:syllables:dde0a4:006
+introclass:syllables:dde0a4:007
+introclass:syllables:dde0a4:008
+introclass:syllables:dde0a4:009
+introclass:syllables:e9c74e:000
+introclass:syllables:f5b56c:000
+introclass:syllables:f5b56c:001
+introclass:syllables:f5b56c:002
+introclass:syllables:f5b56c:003
+introclass:syllables:f8d57d:002
+introclass:syllables:fcf701:002
+introclass:syllables:fcf701:003
+introclass:syllables:fe9d5f:000

python/data/redis/startServer.sh

@@ -0,0 +1,6 @@
+#!/bin/bash 
+#source activate redisEnv
+
+#redis-server --dir $1 --dbfilename $2 --port $3
+redis-server $1/redis.conf --dir $1 --dbfilename $2 --port $3 --daemonize yes
+

python/data/redis/stopServer.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+source activate redisEnv
+redis-cli -p $1 shutdown save

python/data/redisImport.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+file=$1
+port=$2
+pairName=$3
+awk -F, '{ print "SADD pairs", "\"'$pairName'_"$1"_"$2"\"" }' "$file" | sed s/$/$'\r'/ | redis-cli -p $port --pipe #--pipe-timeout 0

python/data/redisSingleImport.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+file=$1
+port=$2
+pairName=$3
+awk -F, '{ print "SADD", "compare","\"'$pairName'/"$1"/"$2"\"",0 }' "$file" | sed s/$/$'\r'/ | redis-cli -p $port --pipe #--pipe-timeout 0

python/data/startBugzoo.sh

@@ -0,0 +1,5 @@
+#!/bin/bash 
+source activate fixminerEnv
+
+bugzood -p $1
+

python/data/statsactions.csv

@@ -0,0 +1,425 @@
+cluster,memberCount
+ExpressionStatement-3-0-0,539
+ExpressionStatement-4-3-0,200
+VariableDeclarationStatement-3-0-0,191
+FieldDeclaration-2-0-0,185
+FieldDeclaration-6-0-0,182
+FieldDeclaration-8-0-0,158
+ExpressionStatement-2-0-0,148
+ExpressionStatement-2-0-1,134
+VariableDeclarationStatement-5-2-0,114
+VariableDeclarationStatement-2-0-0,109
+MethodDeclaration-2-0-0,106
+VariableDeclarationStatement-7-0-0,105
+FieldDeclaration-3-0-0,99
+ExpressionStatement-5-6-0,96
+VariableDeclarationStatement-4-0-0,90
+IfStatement-6-0-0,81
+ExpressionStatement-2-0-2,77
+ExpressionStatement-7-1-0,72
+SwitchCase-2-0-0,70
+MethodDeclaration-2-0-1,66
+ReturnStatement-2-0-0,62
+TypeDeclaration-5-0-0,61
+ReturnStatement-2-0-1,61
+FieldDeclaration-2-0-1,60
+FieldDeclaration-5-2-0,56
+ReturnStatement-2-0-3,56
+CatchClause-3-0-0,54
+MethodDeclaration-14-0-0,53
+ExpressionStatement-2-0-3,52
+FieldDeclaration-11-0-0,48
+ReturnStatement-3-3-0,46
+IfStatement-5-3-0,46
+VariableDeclarationStatement-7-0-2,45
+ReturnStatement-2-0-2,45
+TypeDeclaration-2-1-1,44
+MethodDeclaration-6-2-0,44
+ReturnStatement-4-2-0,42
+VariableDeclarationStatement-7-0-3,42
+ThrowStatement-4-0-0,41
+MethodDeclaration-2-0-2,36
+ExpressionStatement-5-6-1,35
+VariableDeclarationStatement-4-0-1,35
+VariableDeclarationStatement-11-0-0,30
+TypeDeclaration-4-0-0,30
+IfStatement-3-1-0,30
+VariableDeclarationStatement-5-2-2,30
+ThrowStatement-3-1-0,30
+MethodDeclaration-4-1-0,29
+VariableDeclarationStatement-7-0-6,28
+FieldDeclaration-9-0-1,28
+ReturnStatement-4-2-1,28
+VariableDeclarationStatement-7-0-1,27
+SwitchCase-4-0-0,27
+TypeDeclaration-2-1-0,26
+SwitchCase-2-0-1,26
+MethodDeclaration-3-2-0,26
+IfStatement-5-3-1,25
+FieldDeclaration-7-4-0,24
+FieldDeclaration-2-0-2,24
+FieldDeclaration-10-0-0,23
+ExpressionStatement-5-6-4,22
+FieldDeclaration-6-0-1,22
+ExpressionStatement-5-6-2,21
+VariableDeclarationStatement-5-2-1,21
+ReturnStatement-2-0-4,21
+VariableDeclarationStatement-7-0-5,20
+VariableDeclarationStatement-2-0-1,20
+IfStatement-4-4-0,20
+ExpressionStatement-2-0-5,19
+FieldDeclaration-8-0-1,19
+MethodDeclaration-11-1-0,18
+ReturnStatement-2-0-6,18
+ExpressionStatement-6-2-0,17
+ExpressionStatement-5-6-3,16
+VariableDeclarationStatement-5-2-4,16
+MethodDeclaration-13-0-0,16
+ExpressionStatement-2-0-8,15
+SwitchCase-2-0-2,15
+ExpressionStatement-6-2-2,15
+FieldDeclaration-5-2-1,15
+ExpressionStatement-3-0-1,15
+ReturnStatement-2-0-5,14
+TypeDeclaration-2-1-2,14
+ExpressionStatement-2-0-10,14
+VariableDeclarationStatement-11-0-1,14
+ExpressionStatement-4-3-1,14
+VariableDeclarationStatement-10-3-0,14
+ExpressionStatement-6-2-3,14
+VariableDeclarationStatement-5-2-3,14
+ExpressionStatement-2-0-6,13
+VariableDeclarationStatement-7-0-9,13
+TryStatement-7-0-0,13
+ExpressionStatement-2-0-4,13
+EnhancedForStatement-2-0-1,12
+IfStatement-8-5-0,12
+VariableDeclarationStatement-7-0-4,12
+IfStatement-7-5-1,11
+VariableDeclarationStatement-8-11-1,11
+TryStatement-46-0-0,11
+ThrowStatement-2-0-0,11
+ThrowStatement-5-2-1,11
+SynchronizedStatement-4-0-0,11
+SynchronizedStatement-2-0-0,11
+TypeDeclaration-3-1-0,11
+ExpressionStatement-9-2-0,11
+TypeDeclaration-5-0-1,10
+TryStatement-2-0-0,10
+ReturnStatement-7-4-0,9
+IfStatement-9-0-2,9
+CatchClause-6-2-0,9
+EnhancedForStatement-5-1-0,9
+VariableDeclarationStatement-4-0-4,9
+VariableDeclarationStatement-4-0-3,9
+MethodDeclaration-17-0-0,9
+ReturnStatement-2-0-7,9
+MethodDeclaration-4-1-1,8
+ThrowStatement-6-0-0,8
+IfStatement-13-3-0,8
+VariableDeclarationStatement-6-3-0,8
+MethodDeclaration-7-2-0,8
+ThrowStatement-2-0-1,8
+TypeDeclaration-6-0-0,8
+ExpressionStatement-6-2-1,8
+EnhancedForStatement-3-0-0,8
+ExpressionStatement-2-0-9,8
+FieldDeclaration-12-0-0,8
+VariableDeclarationStatement-4-0-6,7
+ReturnStatement-6-2-0,7
+CatchClause-4-0-1,7
+ExpressionStatement-6-2-5,7
+IfStatement-4-4-1,7
+FieldDeclaration-13-0-0,7
+ReturnStatement-4-2-2,7
+SynchronizedStatement-5-0-0,6
+TryStatement-21-0-0,6
+IfStatement-9-0-1,6
+VariableDeclarationStatement-19-1-0,6
+IfStatement-6-0-1,6
+TryStatement-5-0-0,6
+MethodDeclaration-53-0-0,6
+VariableDeclarationStatement-11-0-2,6
+ThrowStatement-5-2-0,6
+ExpressionStatement-6-2-4,6
+VariableDeclarationStatement-14-2-0,6
+SuperConstructorInvocation-2-0-0,6
+EnhancedForStatement-6-1-0,6
+CatchClause-11-0-0,6
+SwitchCase-3-0-0,6
+EnhancedForStatement-7-0-0,6
+EnhancedForStatement-2-0-0,5
+ThrowStatement-5-2-2,5
+FieldDeclaration-9-0-2,5
+IfStatement-12-7-0,5
+TryStatement-10-2-0,5
+IfStatement-2-4-0,5
+FieldDeclaration-19-0-0,5
+FieldDeclaration-11-0-1,5
+VariableDeclarationStatement-7-0-8,5
+ExpressionStatement-2-0-7,5
+ExpressionStatement-2-0-13,5
+ExpressionStatement-13-11-0,5
+CatchClause-13-0-0,5
+ExpressionStatement-4-3-2,5
+VariableDeclarationStatement-4-0-7,5
+ThrowStatement-2-0-2,5
+MethodDeclaration-8-1-0,5
+AssertStatement-2-0-0,5
+MethodDeclaration-10-3-0,5
+VariableDeclarationStatement-7-0-7,5
+ExpressionStatement-8-25-0,5
+VariableDeclarationStatement-3-0-1,5
+CatchClause-4-0-2,5
+CatchClause-16-1-0,4
+CatchClause-8-0-0,4
+ForStatement-22-1-0,4
+ForStatement-27-0-0,4
+MethodDeclaration-32-0-0,4
+CatchClause-9-1-0,4
+FieldDeclaration-7-4-1,4
+IfStatement-14-4-0,4
+ExpressionStatement-5-6-6,4
+ExpressionStatement-5-6-5,4
+MethodDeclaration-23-0-0,4
+IfStatement-53-0-0,4
+SwitchCase-5-0-0,4
+WhileStatement-3-0-0,4
+SynchronizedStatement-11-0-0,4
+VariableDeclarationStatement-4-0-8,4
+EnhancedForStatement-4-1-0,4
+ReturnStatement-5-15-1,4
+VariableDeclarationStatement-18-0-0,4
+VariableDeclarationStatement-4-0-2,4
+VariableDeclarationStatement-4-0-5,4
+FieldDeclaration-15-0-0,4
+IfStatement-7-5-2,4
+TryStatement-4-2-0,4
+TryStatement-28-0-0,4
+FieldDeclaration-9-0-0,4
+ExpressionStatement-28-0-0,4
+IfStatement-10-30-0,4
+TypeDeclaration-4-0-1,4
+VariableDeclarationStatement-5-2-5,4
+MethodDeclaration-15-1-0,4
+IfStatement-37-0-0,4
+MethodDeclaration-20-4-0,4
+MethodDeclaration-5-8-0,4
+FieldDeclaration-14-4-0,4
+ExpressionStatement-23-0-0,3
+EnumDeclaration-9-0-0,3
+ExpressionStatement-2-0-11,3
+MethodDeclaration-16-5-0,3
+ExpressionStatement-12-6-0,3
+ExpressionStatement-3-0-3,3
+ExpressionStatement-3-0-4,3
+MethodDeclaration-12-9-0,3
+ExpressionStatement-8-25-1,3
+MethodDeclaration-22-2-0,3
+MethodDeclaration-46-0-0,3
+IfStatement-9-0-3,3
+ExpressionStatement-2-0-14,3
+TypeDeclaration-8-0-0,3
+ExpressionStatement-10-9-0,3
+IfStatement-9-0-0,3
+EnhancedForStatement-18-0-0,3
+IfStatement-69-0-0,3
+SwitchStatement-2-0-0,3
+SynchronizedStatement-2-0-1,3
+SynchronizedStatement-3-5-0,3
+ThrowStatement-8-0-0,3
+ReturnStatement-5-15-0,3
+ReturnStatement-15-0-0,3
+ReturnStatement-4-2-3,3
+FieldDeclaration-6-0-2,3
+TryStatement-3-0-0,3
+FieldDeclaration-17-0-0,3
+CatchClause-6-2-1,3
+CatchClause-7-0-0,3
+CatchClause-20-0-0,3
+FieldDeclaration-4-3-0,3
+VariableDeclarationStatement-5-2-7,3
+VariableDeclarationStatement-13-2-0,3
+VariableDeclarationStatement-12-1-0,3
+VariableDeclarationStatement-15-2-0,3
+FieldDeclaration-3-0-2,3
+VariableDeclarationStatement-8-11-0,3
+IfStatement-7-5-0,3
+AssertStatement-3-0-0,3
+IfStatement-3-1-1,3
+IfStatement-19-1-0,3
+WhileStatement-12-0-0,3
+ForStatement-3-0-0,3
+ExpressionStatement-14-2-0,3
+TryStatement-16-0-0,3
+IfStatement-15-1-0,3
+TryStatement-8-0-0,3
+IfStatement-46-0-0,3
+TryStatement-38-0-0,3
+ExpressionStatement-19-4-0,3
+IfStatement-17-6-0,3
+ThrowStatement-9-1-0,2
+TryStatement-6-0-0,2
+TryStatement-14-0-0,2
+ExpressionStatement-24-0-0,2
+ExpressionStatement-2-0-15,2
+ExpressionStatement-21-0-0,2
+TryStatement-11-4-0,2
+ExpressionStatement-8-25-2,2
+ExpressionStatement-2-0-12,2
+FieldDeclaration-4-3-1,2
+TryStatement-18-2-0,2
+TryStatement-20-0-0,2
+ExpressionStatement-26-1-0,2
+TryStatement-24-0-0,2
+TryStatement-13-1-0,2
+FieldDeclaration-3-0-1,2
+ExpressionStatement-5-6-7,2
+TryStatement-12-1-0,2
+TryStatement-6-0-1,2
+FieldDeclaration-6-0-3,2
+ExpressionStatement-4-3-3,2
+ThrowStatement-10-0-0,2
+ExpressionStatement-3-0-6,2
+TryStatement-8-0-1,2
+ExpressionStatement-2-0-16,2
+ExpressionStatement-15-3-0,2
+TryStatement-25-0-0,2
+ThrowStatement-7-5-0,2
+ExpressionStatement-3-0-2,2
+ExpressionStatement-2-0-17,2
+FieldDeclaration-16-0-0,2
+ThrowStatement-12-0-0,2
+ExpressionStatement-3-0-5,2
+ThrowStatement-2-0-3,2
+ExpressionStatement-2-0-18,2
+TryStatement-6-0-2,2
+FieldDeclaration-18-0-0,2
+ExpressionStatement-2-0-19,2
+TryStatement-15-0-0,2
+VariableDeclarationStatement-20-0-0,2
+ExpressionStatement-17-3-0,2
+SynchronizedStatement-12-0-0,2
+ReturnStatement-10-5-0,2
+ReturnStatement-8-9-0,2
+ReturnStatement-23-0-0,2
+ReturnStatement-4-2-4,2
+ReturnStatement-12-2-0,2
+ReturnStatement-2-0-8,2
+ReturnStatement-13-1-0,2
+ReturnStatement-5-15-2,2
+ReturnStatement-14-0-0,2
+ReturnStatement-22-0-0,2
+SynchronizedStatement-9-0-0,2
+SynchronizedStatement-7-0-0,2
+SynchronizedStatement-6-3-0,2
+IfStatement-20-2-0,2
+IfStatement-21-1-0,2
+IfStatement-18-10-0,2
+IfStatement-27-1-0,2
+IfStatement-11-0-0,2
+IfStatement-11-0-1,2
+IfStatement-11-0-2,2
+IfStatement-7-5-4,2
+IfStatement-7-5-3,2
+IfStatement-29-0-0,2
+IfStatement-16-1-0,2
+IfStatement-74-0-0,2
+IfStatement-6-0-3,2
+IfStatement-6-0-2,2
+IfStatement-28-1-0,2
+ReturnStatement-11-2-0,2
+ReturnStatement-9-3-0,2
+TypeDeclaration-5-0-2,2
+TypeDeclaration-6-0-1,2
+VariableDeclarationStatement-9-34-0,2
+VariableDeclarationStatement-11-0-3,2
+VariableDeclarationStatement-7-0-10,2
+VariableDeclarationStatement-16-8-0,2
+VariableDeclarationStatement-17-0-0,2
+VariableDeclarationStatement-8-11-4,2
+VariableDeclarationStatement-8-11-3,2
+VariableDeclarationStatement-8-11-2,2
+VariableDeclarationStatement-4-0-9,2
+VariableDeclarationStatement-3-0-2,2
+VariableDeclarationStatement-5-2-9,2
+VariableDeclarationStatement-5-2-6,2
+VariableDeclarationStatement-5-2-8,2
+SuperConstructorInvocation-6-0-0,2
+SuperConstructorInvocation-4-0-0,2
+SuperConstructorInvocation-2-0-1,2
+CatchClause-29-0-0,2
+CatchClause-17-1-0,2
+CatchClause-10-1-0,2
+CatchClause-19-0-0,2
+CatchClause-24-0-0,2
+CatchClause-4-0-0,2
+CatchClause-12-0-0,2
+CatchClause-2-0-0,2
+CatchClause-5-1-0,2
+CatchClause-14-0-0,2
+TypeDeclaration-7-1-0,2
+IfStatement-26-1-0,2
+IfStatement-31-0-0,2
+ExpressionStatement-6-2-8,2
+MethodDeclaration-54-0-0,2
+MethodDeclaration-56-0-0,2
+MethodDeclaration-51-0-0,2
+MethodDeclaration-33-0-0,2
+MethodDeclaration-18-0-0,2
+MethodDeclaration-27-1-0,2
+MethodDeclaration-9-15-0,2
+MethodDeclaration-11-1-1,2
+MethodDeclaration-29-0-0,2
+MethodDeclaration-42-0-0,2
+MethodDeclaration-6-2-1,2
+MethodDeclaration-19-2-0,2
+MethodDeclaration-26-0-0,2
+MethodDeclaration-21-0-0,2
+MethodDeclaration-39-0-0,2
+IfStatement-79-0-0,2
+MethodDeclaration-90-0-0,2
+MethodDeclaration-41-0-0,2
+MethodDeclaration-24-0-0,2
+MethodDeclaration-13-0-1,2
+MethodDeclaration-14-0-1,2
+MethodDeclaration-25-0-0,2
+ExpressionStatement-35-0-0,2
+ExpressionStatement-20-0-0,2
+ExpressionStatement-18-0-0,2
+ExpressionStatement-11-42-0,2
+ExpressionStatement-16-4-0,2
+ExpressionStatement-6-2-7,2
+ExpressionStatement-6-2-6,2
+MethodDeclaration-35-1-0,2
+EnumDeclaration-2-0-0,2
+EnumDeclaration-8-0-0,2
+EnumDeclaration-11-0-0,2
+IfStatement-24-0-0,2
+IfStatement-23-1-0,2
+IfStatement-5-3-3,2
+IfStatement-5-3-2,2
+IfStatement-22-1-0,2
+IfStatement-25-0-0,2
+ForStatement-16-0-0,2
+ForStatement-80-0-0,2
+ForStatement-19-0-0,2
+ForStatement-8-0-0,2
+ForStatement-24-0-0,2
+ForStatement-4-0-0,2
+ForStatement-13-0-0,2
+WhileStatement-18-0-0,2
+WhileStatement-7-0-0,2
+WhileStatement-6-0-0,2
+WhileStatement-17-0-0,2
+WhileStatement-8-0-0,2
+EnhancedForStatement-20-0-0,2
+EnhancedForStatement-27-0-0,2
+EnhancedForStatement-16-1-0,2
+EnhancedForStatement-10-2-0,2
+EnhancedForStatement-12-0-0,2
+EnhancedForStatement-13-1-0,2
+EnhancedForStatement-14-1-0,2
+ConstructorInvocation-13-0-0,2
+ConstructorInvocation-5-0-0,2
+WhileStatement-2-0-0,2

python/data/statstokens.csv

@@ -0,0 +1,307 @@
+cluster,memberCount
+VariableDeclarationStatement-2-0-0-0,108
+FieldDeclaration-2-0-0-1,81
+ExpressionStatement-2-0-2-0,66
+IfStatement-6-0-0-0,65
+FieldDeclaration-2-0-0-2,50
+FieldDeclaration-2-0-0-0,33
+ExpressionStatement-3-0-0-2,32
+MethodDeclaration-2-0-1-3,24
+MethodDeclaration-2-0-1-0,17
+CatchClause-3-0-0-0,15
+SwitchCase-2-0-1-0,13
+ExpressionStatement-3-0-0-8,13
+FieldDeclaration-2-0-0-3,12
+ExpressionStatement-3-0-0-4,12
+MethodDeclaration-2-0-1-1,12
+TypeDeclaration-2-1-1-0,10
+ExpressionStatement-3-0-0-0,9
+ExpressionStatement-3-0-0-1,9
+VariableDeclarationStatement-10-3-0-0,8
+ExpressionStatement-4-3-0-0,8
+TryStatement-46-0-0-0,8
+ExpressionStatement-3-0-0-6,8
+ReturnStatement-2-0-2-1,8
+ThrowStatement-6-0-0-0,8
+ExpressionStatement-3-0-0-9,8
+ExpressionStatement-3-0-0-3,7
+VariableDeclarationStatement-5-2-4-0,7
+ExpressionStatement-3-0-0-13,7
+ThrowStatement-3-1-0-0,6
+ExpressionStatement-5-6-0-1,6
+TryStatement-21-0-0-0,6
+ExpressionStatement-2-0-2-1,6
+IfStatement-6-0-0-1,6
+FieldDeclaration-2-0-0-5,6
+VariableDeclarationStatement-19-1-0-0,6
+SwitchCase-3-0-0-0,6
+ExpressionStatement-3-0-0-12,6
+MethodDeclaration-2-0-1-2,6
+SynchronizedStatement-4-0-0-0,6
+CatchClause-11-0-0-0,6
+ReturnStatement-6-2-0-0,5
+MethodDeclaration-3-2-0-0,5
+ExpressionStatement-3-0-0-28,5
+ExpressionStatement-5-6-0-0,5
+ExpressionStatement-3-0-0-11,5
+VariableDeclarationStatement-11-0-1-0,5
+ExpressionStatement-6-2-4-0,5
+ExpressionStatement-3-0-0-15,5
+ExpressionStatement-13-11-0-0,5
+ExpressionStatement-5-6-0-2,4
+ExpressionStatement-3-0-0-27,4
+MethodDeclaration-23-0-0-0,4
+ExpressionStatement-3-0-0-20,4
+ExpressionStatement-3-0-0-7,4
+ReturnStatement-2-0-2-0,4
+ExpressionStatement-3-0-0-31,4
+MethodDeclaration-2-0-1-4,4
+ReturnStatement-7-4-0-0,4
+CatchClause-3-0-0-2,4
+CatchClause-3-0-0-4,4
+FieldDeclaration-9-0-0-0,4
+ThrowStatement-5-2-1-1,4
+ThrowStatement-5-2-1-0,4
+MethodDeclaration-5-8-0-0,4
+CatchClause-8-0-0-0,4
+VariableDeclarationStatement-3-0-0-2,4
+CatchClause-16-1-0-0,4
+ExpressionStatement-10-9-0-0,3
+ReturnStatement-5-15-0-0,3
+ExpressionStatement-19-4-0-0,3
+ExpressionStatement-9-2-0-0,3
+FieldDeclaration-6-0-0-0,3
+SwitchCase-4-0-0-1,3
+ExpressionStatement-3-0-0-25,3
+SynchronizedStatement-4-0-0-2,3
+SwitchStatement-2-0-0-0,3
+VariableDeclarationStatement-14-2-0-0,3
+VariableDeclarationStatement-15-2-0-0,3
+FieldDeclaration-6-0-0-5,3
+ExpressionStatement-3-0-0-17,3
+ExpressionStatement-3-0-0-10,3
+VariableDeclarationStatement-4-0-0-0,3
+IfStatement-17-6-0-0,3
+IfStatement-10-30-0-0,3
+WhileStatement-12-0-0-0,3
+ExpressionStatement-3-0-0-30,3
+IfStatement-3-1-0-1,3
+ExpressionStatement-3-0-0-26,3
+VariableDeclarationStatement-10-3-0-2,3
+FieldDeclaration-2-0-0-4,3
+ExpressionStatement-3-0-0-14,3
+ExpressionStatement-2-0-2-2,3
+TryStatement-16-0-0-0,3
+ExpressionStatement-3-0-0-18,3
+VariableDeclarationStatement-10-3-0-1,3
+MethodDeclaration-3-2-0-1,3
+ReturnStatement-7-4-0-2,3
+TryStatement-46-0-0-1,3
+ThrowStatement-3-1-0-5,3
+VariableDeclarationStatement-14-2-0-1,3
+TryStatement-38-0-0-0,3
+ExpressionStatement-3-0-0-43,3
+MethodDeclaration-2-0-1-5,3
+CatchClause-6-2-0-0,3
+ReturnStatement-15-0-0-0,3
+TypeDeclaration-2-1-1-1,3
+ThrowStatement-3-1-0-3,3
+ThrowStatement-3-1-0-4,3
+CatchClause-3-0-0-8,3
+ExpressionStatement-5-6-0-5,3
+CatchClause-7-0-0-0,3
+ExpressionStatement-5-6-0-4,3
+ThrowStatement-2-0-1-0,3
+ExpressionStatement-5-6-0-6,3
+MethodDeclaration-3-2-0-3,3
+ExpressionStatement-3-0-0-50,2
+ExpressionStatement-3-0-0-19,2
+ExpressionStatement-3-0-0-59,2
+ExpressionStatement-3-0-0-21,2
+ExpressionStatement-3-0-0-42,2
+ExpressionStatement-3-0-0-57,2
+ExpressionStatement-3-0-0-45,2
+ExpressionStatement-3-0-0-32,2
+ExpressionStatement-3-0-0-35,2
+ExpressionStatement-3-0-0-56,2
+ExpressionStatement-3-0-0-51,2
+ExpressionStatement-3-0-0-58,2
+ExpressionStatement-3-0-0-16,2
+ExpressionStatement-3-0-0-29,2
+ExpressionStatement-3-0-0-34,2
+ExpressionStatement-3-0-0-33,2
+ExpressionStatement-3-0-0-44,2
+VariableDeclarationStatement-20-0-0-0,2
+ExpressionStatement-3-0-0-38,2
+TryStatement-5-0-0-0,2
+ThrowStatement-7-5-0-0,2
+ThrowStatement-4-0-0-0,2
+ThrowStatement-3-1-0-6,2
+ThrowStatement-3-1-0-1,2
+ThrowStatement-3-1-0-2,2
+ThrowStatement-12-0-0-0,2
+FieldDeclaration-11-0-0-0,2
+FieldDeclaration-6-0-0-9,2
+FieldDeclaration-6-0-0-11,2
+FieldDeclaration-6-0-0-7,2
+FieldDeclaration-6-0-0-6,2
+FieldDeclaration-6-0-0-1,2
+FieldDeclaration-6-0-0-10,2
+FieldDeclaration-6-0-0-8,2
+FieldDeclaration-6-0-0-4,2
+FieldDeclaration-6-0-0-3,2
+FieldDeclaration-6-0-0-12,2
+FieldDeclaration-6-0-0-2,2
+FieldDeclaration-17-0-0-0,2
+FieldDeclaration-3-0-0-0,2
+FieldDeclaration-3-0-0-1,2
+ThrowStatement-9-1-0-0,2
+TryStatement-3-0-0-0,2
+ExpressionStatement-3-0-0-36,2
+TryStatement-15-0-0-0,2
+ExpressionStatement-3-0-0-54,2
+ExpressionStatement-3-0-0-53,2
+ExpressionStatement-3-0-0-37,2
+ExpressionStatement-3-0-0-39,2
+ExpressionStatement-3-0-0-52,2
+ExpressionStatement-3-0-0-55,2
+ExpressionStatement-3-0-0-46,2
+ExpressionStatement-3-0-0-41,2
+ExpressionStatement-3-0-0-48,2
+ExpressionStatement-3-0-0-24,2
+ExpressionStatement-3-0-0-23,2
+ExpressionStatement-3-0-0-49,2
+ExpressionStatement-3-0-0-40,2
+ExpressionStatement-3-0-0-47,2
+ExpressionStatement-3-0-0-5,2
+ExpressionStatement-3-0-0-22,2
+ExpressionStatement-12-6-0-0,2
+ExpressionStatement-4-3-0-5,2
+ExpressionStatement-5-6-0-7,2
+ExpressionStatement-5-6-0-3,2
+TryStatement-4-2-0-0,2
+ExpressionStatement-15-3-0-0,2
+MethodDeclaration-4-1-0-2,2
+ExpressionStatement-4-3-0-2,2
+CatchClause-3-0-0-7,2
+CatchClause-3-0-0-1,2
+CatchClause-3-0-0-3,2
+CatchClause-3-0-0-5,2
+CatchClause-14-0-0-0,2
+SwitchCase-4-0-0-0,2
+TypeDeclaration-3-1-0-0,2
+TypeDeclaration-2-1-1-3,2
+TypeDeclaration-2-1-1-2,2
+TypeDeclaration-5-0-0-0,2
+ReturnStatement-9-3-0-0,2
+ReturnStatement-11-2-0-0,2
+ReturnStatement-7-4-0-1,2
+ReturnStatement-6-2-0-1,2
+ReturnStatement-23-0-0-0,2
+ReturnStatement-4-2-2-0,2
+ReturnStatement-4-2-2-1,2
+ReturnStatement-3-3-0-20,2
+ReturnStatement-3-3-0-18,2
+ReturnStatement-3-3-0-9,2
+ReturnStatement-3-3-0-0,2
+ReturnStatement-3-3-0-11,2
+ReturnStatement-3-3-0-7,2
+ReturnStatement-3-3-0-16,2
+ReturnStatement-3-3-0-6,2
+ReturnStatement-3-3-0-17,2
+ReturnStatement-3-3-0-1,2
+ReturnStatement-3-3-0-10,2
+CatchClause-3-0-0-6,2
+CatchClause-3-0-0-9,2
+ReturnStatement-3-3-0-8,2
+CatchClause-4-0-1-0,2
+VariableDeclarationStatement-18-0-0-1,2
+VariableDeclarationStatement-7-0-3-0,2
+VariableDeclarationStatement-7-0-3-1,2
+VariableDeclarationStatement-7-0-3-3,2
+VariableDeclarationStatement-7-0-3-2,2
+VariableDeclarationStatement-6-3-0-0,2
+VariableDeclarationStatement-8-11-4-0,2
+VariableDeclarationStatement-4-0-0-1,2
+VariableDeclarationStatement-4-0-0-3,2
+VariableDeclarationStatement-4-0-0-2,2
+VariableDeclarationStatement-3-0-0-9,2
+VariableDeclarationStatement-3-0-0-0,2
+VariableDeclarationStatement-3-0-0-11,2
+VariableDeclarationStatement-3-0-0-7,2
+VariableDeclarationStatement-3-0-0-6,2
+VariableDeclarationStatement-3-0-0-1,2
+VariableDeclarationStatement-3-0-0-10,2
+VariableDeclarationStatement-3-0-0-8,2
+VariableDeclarationStatement-3-0-0-4,2
+VariableDeclarationStatement-3-0-0-3,2
+VariableDeclarationStatement-3-0-0-12,2
+VariableDeclarationStatement-3-0-0-13,2
+VariableDeclarationStatement-3-0-0-5,2
+VariableDeclarationStatement-12-1-0-0,2
+SuperConstructorInvocation-6-0-0-0,2
+SuperConstructorInvocation-4-0-0-0,2
+CatchClause-9-1-0-0,2
+ReturnStatement-3-3-0-19,2
+ReturnStatement-3-3-0-21,2
+ExpressionStatement-4-3-0-3,2
+ForStatement-19-0-0-0,2
+WhileStatement-8-0-0-0,2
+EnhancedForStatement-16-1-0-0,2
+EnhancedForStatement-4-1-0-0,2
+EnhancedForStatement-3-0-0-0,2
+EnhancedForStatement-2-0-1-0,2
+EnhancedForStatement-2-0-1-1,2
+EnhancedForStatement-14-1-0-0,2
+ConstructorInvocation-13-0-0-0,2
+ConstructorInvocation-5-0-0-0,2
+MethodDeclaration-35-1-0-0,2
+MethodDeclaration-42-0-0-0,2
+MethodDeclaration-6-2-0-0,2
+MethodDeclaration-54-0-0-0,2
+MethodDeclaration-4-1-0-0,2
+MethodDeclaration-4-1-0-1,2
+VariableDeclarationStatement-18-0-0-0,2
+MethodDeclaration-3-2-0-2,2
+MethodDeclaration-12-9-0-0,2
+MethodDeclaration-14-0-0-0,2
+ExpressionStatement-9-2-0-1,2
+ExpressionStatement-11-42-0-0,2
+ExpressionStatement-16-4-0-0,2
+ExpressionStatement-26-1-0-0,2
+ExpressionStatement-8-25-2-0,2
+ExpressionStatement-4-3-0-6,2
+ExpressionStatement-4-3-0-1,2
+ExpressionStatement-4-3-0-4,2
+WhileStatement-17-0-0-0,2
+IfStatement-22-1-0-0,2
+ReturnStatement-3-3-0-4,2
+IfStatement-5-3-0-2,2
+ReturnStatement-3-3-0-15,2
+ReturnStatement-3-3-0-3,2
+ReturnStatement-3-3-0-12,2
+ReturnStatement-3-3-0-2,2
+ReturnStatement-3-3-0-13,2
+ReturnStatement-3-3-0-5,2
+ReturnStatement-3-3-0-14,2
+ReturnStatement-3-3-0-22,2
+ReturnStatement-2-0-2-2,2
+ReturnStatement-13-1-0-0,2
+SynchronizedStatement-7-0-0-0,2
+SynchronizedStatement-4-0-0-1,2
+SynchronizedStatement-12-0-0-0,2
+SynchronizedStatement-2-0-0-0,2
+IfStatement-18-10-0-0,2
+IfStatement-9-0-1-0,2
+IfStatement-11-0-2-0,2
+IfStatement-29-0-0-0,2
+IfStatement-74-0-0-0,2
+IfStatement-19-1-0-0,2
+IfStatement-4-4-1-0,2
+IfStatement-3-1-0-0,2
+IfStatement-3-1-0-2,2
+IfStatement-13-3-0-0,2
+IfStatement-5-3-0-0,2
+IfStatement-5-3-0-1,2
+IfStatement-5-3-0-3,2
+FieldDeclaration-5-2-1-0,2

python/data/subjects.csv

@@ -0,0 +1,47 @@
+Group,Subject,Repo,GitRepo,Branch
+Apache,CAMEL,camel,https://github.com/apache/camel.git,master
+Apache,HBASE,hbase,https://github.com/apache/hbase.git,master
+Apache,HIVE,hive,https://github.com/apache/hive.git,master
+Commons,CODEC,commons-codec,https://github.com/apache/commons-codec.git,trunk
+Commons,COLLECTIONS,commons-collections,https://github.com/apache/commons-collections.git,master
+Commons,COMPRESS,commons-compress,https://github.com/apache/commons-compress.git,master
+Commons,CONFIGURATION,commons-configuration,https://github.com/apache/commons-configuration.git,trunk
+Commons,CRYPTO,commons-crypto,https://github.com/apache/commons-crypto.git,master
+Commons,CSV,commons-csv,https://github.com/apache/commons-csv.git,master
+Commons,IO,commons-io,https://github.com/apache/commons-io.git,master
+Commons,WEAVER,commons-weaver,https://github.com/apache/commons-weaver.git,master
+JBoss,ENTESB,fuse,https://github.com/jboss-fuse/fuse.git,6.3.0.redhat
+JBoss,JBMETA,metadata,https://github.com/jboss/metadata.git,master
+Wildfly,ELY,wildfly-elytron,https://github.com/wildfly-security/wildfly-elytron.git,master
+Wildfly,SWARM,wildfly-swarm,https://github.com/wildfly-swarm/wildfly-swarm.git,master
+Wildfly,WFARQ,wildfly-arquillian,https://github.com/wildfly/wildfly-arquillian.git,master
+Wildfly,WFCORE,wildfly-core,https://github.com/wildfly/wildfly-core.git,master
+Wildfly,WFLY,wildfly,https://github.com/wildfly/wildfly.git,master
+Wildfly,WFMP,wildfly-maven-plugin,https://github.com/wildfly/wildfly-maven-plugin.git,master
+Spring,AMQP,spring-amqp,https://github.com/spring-projects/spring-amqp,master
+Spring,ANDROID,spring-android,https://github.com/spring-projects/spring-android,master
+Spring,BATCH,spring-batch,https://github.com/spring-projects/spring-batch,master
+Spring,BATCHADM,spring-batch-admin,https://github.com/spring-projects/spring-batch-admin,master
+Spring,DATACMNS,spring-data-commons,https://github.com/spring-projects/spring-data-commons,master
+Spring,DATAGRAPH,spring-data-neo4j,https://github.com/spring-projects/spring-data-neo4j,master
+Spring,DATAJPA,spring-data-jpa,https://github.com/spring-projects/spring-data-jpa,master
+Spring,DATAMONGO,spring-data-mongodb,https://github.com/spring-projects/spring-data-mongodb,master
+Spring,DATAREDIS,spring-data-redis,https://github.com/spring-projects/spring-data-redis,master
+Spring,DATAREST,spring-data-rest,https://github.com/spring-projects/spring-data-rest,master
+Spring,LDAP,spring-ldap,https://github.com/spring-projects/spring-ldap,master
+Spring,MOBILE,spring-mobile,https://github.com/spring-projects/spring-mobile,master
+Spring,ROO,spring-roo,https://github.com/spring-projects/spring-roo,master
+Spring,SEC,spring-security,https://github.com/spring-projects/spring-security,master
+Spring,SECOAUTH,spring-security-oauth,https://github.com/spring-projects/spring-security-oauth,master
+Spring,SGF,spring-data-gemfire,https://github.com/spring-projects/spring-data-gemfire,master
+Spring,SHDP,spring-hadoop,https://github.com/spring-projects/spring-hadoop,master
+Spring,SHL,spring-shell,https://github.com/spring-projects/spring-shell,master
+Spring,SOCIAL,spring-social,https://github.com/spring-projects/spring-social,master
+Spring,SOCIALFB,spring-social-facebook,https://github.com/spring-projects/spring-social-facebook,master
+Spring,SOCIALLI,spring-social-linkedin,https://github.com/spring-projects/spring-social-linkedin,master
+Spring,SOCIALTW,spring-social-twitter,https://github.com/spring-projects/spring-social-twitter,master
+Spring,SPR,spring-framework,https://github.com/spring-projects/spring-framework,master
+Spring,SWF,spring-webflow,https://github.com/spring-projects/spring-webflow,master
+Spring,SWS,spring-ws,https://github.com/spring-projects/spring-ws,master
+
+

python/dataset.csv

@@ -0,0 +1,50 @@
+repo,bid
+camel,1366
+closure-compiler.git,0
+commons-codec,11
+commons-collections,56
+commons-compress,73
+commons-configuration,89
+commons-crypto,9
+commons-csv,18
+commons-io,58
+commons-lang,129
+commons-lang.git,0
+commons-math,157
+commons-math.git,0
+fuse,15
+hbase,2169
+hive,2641
+metadata,14
+mockito.git,0
+spring-amqp,89
+spring-android,5
+spring-batch,224
+spring-batch-admin,11
+spring-data-commons,151
+spring-data-gemfire,35
+spring-data-jpa,112
+spring-data-mongodb,190
+spring-data-neo4j,19
+spring-data-redis,65
+spring-data-rest,91
+spring-framework,1098
+spring-hadoop,35
+spring-ldap,26
+spring-mobile,11
+spring-roo,414
+spring-security,304
+spring-security-oauth,66
+spring-shell,11
+spring-social,14
+spring-social-facebook,12
+spring-social-linkedin,2
+spring-social-twitter,9
+spring-webflow,84
+spring-ws,101
+wildfly,802
+wildfly-arquillian,8
+wildfly-core,547
+wildfly-elytron,217
+wildfly-maven-plugin,13
+wildfly-swarm,131

python/dataset.py

@@ -0,0 +1,169 @@
+from common.commons import *
+
+DATA_PATH = os.environ["DATA_PATH"]
+REPO_PATH = os.environ["REPO_PATH"]
+def prepareFiles(t):
+    try:
+        sha, repoName = t
+
+        shaOld = sha + '^'
+
+        repo = join(REPO_PATH,repoName)
+        gumInputRepo = join(DATA_PATH, 'gumInput', repoName)
+        if not os.path.exists(join(gumInputRepo)):
+            os.makedirs(gumInputRepo)
+
+        cmd = 'git -C ' + repo + ' diff --name-only ' + shaOld + '..'+sha
+
+        output, errors = shellGitCheckout(cmd, 'latin1')
+        files = output.strip().split('\n')
+
+        # if len(nonJava) > 0:
+        #     logging.warning('Skipping commit %s',sha)
+        #     return
+        # if len(files) != 1:
+        #     return
+
+        nonTest = [f for f in files if not re.search('test', f, re.I) and f.endswith('java')]
+
+        if len(nonTest) > 1:
+            return
+
+        cmd = 'git -C ' + repo + ' rev-parse --short=6 ' + shaOld
+
+        output, errors = shellGitCheckout(cmd, 'latin1')
+        shaOld = output.strip()
+
+        cmd = 'git -C ' + repo + ' rev-parse --short=6 ' + sha
+        output, errors = shellGitCheckout(cmd, 'latin1')
+        sha = output.strip()
+
+        if isinstance(nonTest, list):
+            for file in nonTest:
+                checkoutFiles(sha,shaOld, repoName, file,'gumInput')
+
+
+
+    except Exception as e:
+        print(e)
+
+
+def prepareFilesDefects4J(repo,repoName,shaOld):
+    try:
+        # sha, repoName = t
+
+        sha = shaOld + '^'
+
+        # repo = join(REPO_PATH,repoName)
+        gumInputRepo = join(DATA_PATH, 'Defects4J2', repoName)
+        if not os.path.exists(join(gumInputRepo)):
+            os.makedirs(gumInputRepo)
+
+        cmd = 'git -C ' + repo + ' diff --name-only ' + shaOld + '..'+sha
+        output, errors = shellGitCheckout(cmd, 'latin1')
+        files = output.strip().split('\n')
+        # nonJava = [f for f in files if not f.endswith('.java')]
+        nonTest = [f for f in files if not re.search('test',f,re.I)]
+        # if len(nonJava) > 0:
+        #     logging.warning('Skipping commit %s',sha)
+        #     return
+        # if len(files) != 1:
+        #     return
+
+        nonTest = [f for f in files if not re.search('test', f, re.I) and f.endswith('java')]
+
+        if len(nonTest) > 1:
+            return
+
+        cmd = 'git -C ' + repo + ' rev-parse --short=6 ' + shaOld
+        output, errors = shellGitCheckout(cmd, 'latin1')
+        shaOld = output.strip()
+
+        cmd = 'git -C ' + repo + ' rev-parse --short=6 ' + sha
+        output, errors = shellGitCheckout(cmd, 'latin1')
+        sha = output.strip()
+
+        if isinstance(nonTest, list):
+            for file in nonTest:
+                checkoutFiles(sha,shaOld, repoName, file,'Defects4J2',repo)
+
+
+    except Exception as e:
+        print(e)
+
+
+
+
+def checkoutFiles(sha,shaOld,repoName, filePath,type, repo=None):
+    try:
+        # folderDiff = join(DATA_PATH, 'gumInput',repoName, 'DiffEntries')
+        folderDiff = join(DATA_PATH, type,repoName, 'DiffEntries')
+        folderPrev = join(DATA_PATH, type,repoName, 'prevFiles')
+        folderRev = join(DATA_PATH, type,repoName, 'revFiles')
+        if not os.path.exists(folderDiff):
+            os.mkdir(folderDiff)
+
+        if not os.path.exists(folderPrev):
+            os.mkdir(folderPrev)
+
+        if not os.path.exists(folderRev):
+            os.mkdir(folderRev)
+
+        if repo is None:
+            repo = join(REPO_PATH,repoName)
+
+
+        savePath = filePath.replace('/','#')
+        if not isfile(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java', '.txt')):
+
+            cmd = 'git -C ' + repo + ' diff -U ' + shaOld + ':' + filePath + '..' + sha + ':' + filePath  # + '> ' + folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt')
+
+            output,errors = shellGitCheckout(cmd,'latin1')
+            if errors:
+                # print(errors)
+                raise FileNotFoundError
+
+            regex = r"@@\s\-\d+,*\d*\s\+\d+,*\d*\s@@ ?(.*\n)*"
+            match = re.search(regex, output)
+            if not match:
+                return
+                # print()
+            not_matched, matched = output[:match.start()], match.group()
+            numberOfHunks = re.findall('@@\s\-\d+,*\d*\s\+\d+,*\d*\s@@', matched)
+            if len(numberOfHunks) == 0:
+                return
+            diffFile = shaOld + '\n' + matched.replace(' @@ ', ' @@\n')
+            with open(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java', '.txt'),
+                      'w') as writeFile:
+                writeFile.writelines(diffFile)
+
+
+
+            cmd = 'git -C ' + repo + ' show ' + sha + ':' + filePath + '> ' + folderRev + '/' + sha + '_' + shaOld + '_' +savePath
+
+            if errors:
+                # print(errors)
+                raise FileNotFoundError
+            o,errors= shellGitCheckout(cmd,'latin1')
+            cmd = 'git -C ' + repo + ' show ' + shaOld + ':' + filePath + '> ' + folderPrev + '/' + 'prev_'+sha + '_' + shaOld + '_' +savePath
+            if errors:
+                # print(errors)
+                raise FileNotFoundError
+
+            o,errors = shellGitCheckout(cmd,'latin1')
+            if errors:
+                # print(errors)
+                raise FileNotFoundError
+
+    except FileNotFoundError as fnfe:
+        if isfile(folderRev + '/' + sha + '_' + shaOld + '_' +savePath):
+            os.remove(folderRev + '/' + sha + '_' + shaOld + '_' +savePath)
+        if isfile(folderPrev + '/' + 'prev_'+sha + '_' + shaOld + '_' +savePath):
+            os.remove(folderPrev + '/' + 'prev_'+sha + '_' + shaOld + '_' +savePath)
+        if isfile(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt')):
+            os.remove(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt'))
+        # print(fnfe)
+        # raise Exception(fnfe)
+    except Exception as e:
+        # print(e)
+        raise Exception(e)

python/dataset.tex

@@ -0,0 +1,50 @@
+\begin{tabular}{llr}
+\toprule
+{} &                   repo &   bid \\
+\midrule
+0  &           APACHE-CAMEL &  1366 \\
+1  &          APACHE-COMMONS-CODEC &    11 \\
+2  &    APACHE-COMMONS-COLLECTIONS &    56 \\
+3  &       APACHE-COMMONS-COMPRESS &    73 \\
+4  &  APACHE-COMMONS-CONFIGURATION &    89 \\
+5  &         APACHE-COMMONS-CRYPTO &     9 \\
+6  &            APACHE-COMMONS-CSV &    18 \\
+7  &             APACHE-COMMONS-IO &    58 \\
+9  &           APACHE-HBASE &  2169 \\
+10 &            APACHE-HIVE &  2641 \\
+8  &           JBOSS-ENTESB &    15 \\
+11 &           JBOSS-JBMETA &    14 \\
+12 &            SPRING-AMQP &    89 \\
+13 &         SPRING-ANDROID &     5 \\
+14 &           SPRING-BATCH &   224 \\
+15 &        SPRING-BATCHADM &    11 \\
+16 &        SPRING-DATACMNS &   151 \\
+20 &       SPRING-DATAGRAPH &    19 \\
+18 &         SPRING-DATAJPA &   112 \\
+19 &       SPRING-DATAMONGO &   190 \\
+21 &       SPRING-DATAREDIS &    65 \\
+22 &        SPRING-DATAREST &    91 \\
+25 &            SPRING-LDAP &    26 \\
+26 &          SPRING-MOBILE &    11 \\
+27 &             SPRING-ROO &   414 \\
+28 &             SPRING-SEC &   304 \\
+29 &        SPRING-SECOAUTH &    66 \\
+17 &             SPRING-SGF &    35 \\
+24 &            SPRING-SHDP &    35 \\
+30 &             SPRING-SHL &    11 \\
+31 &          SPRING-SOCIAL &    14 \\
+32 &        SPRING-SOCIALFB &    12 \\
+33 &        SPRING-SOCIALLI &     2 \\
+34 &        SPRING-SOCIALTW &     9 \\
+23 &             SPRING-SPR &  1098 \\
+35 &             SPRING-SWF &    84 \\
+36 &             SPRING-SWS &   101 \\
+40 &            WILDFLY-ELY &   217 \\
+42 &          WILDFLY-SWARM &   131 \\
+38 &          WILDFLY-WFARQ &     8 \\
+39 &         WILDFLY-WFCORE &   547 \\
+37 &           WILDFLY-WFLY &   802 \\
+41 &           WILDFLY-WFMP &    13 \\
+11 &        Total & 1141\\
+\bottomrule
+\end{tabular}

python/defects4JDataset.py

@@ -0,0 +1,33 @@
+from common.commons import *
+
+
+
+
+
+
+def core():
+
+    projects = [('Math',106,'commons-math'),('Lang',65,'commons-lang'),('Closure',133,'closure-compiler'),('Time',27,'joda-time'),('Mockito',38,'mockito'),('Chart',26,'jfreechart')]
+    path = '/Users/anil.koyuncu/KUI/Defects4JData/'
+    with open('mapping.csv','a') as f:
+        for p in projects:
+            pjName,iterSize,repoName = p
+
+            for idx in range(iterSize):
+                idx+=1
+
+                # cmd = '~/bugStudy/defects4j/framework/bin/defects4j info -p ' + pjDict[project]
+                repoPath = join(path,pjName+'_'+str(idx))
+                cmd = 'git -C ' +repoPath+ ' checkout -- .'
+                output, errors = shellGitCheckout(cmd, 'latin1')
+                cmd = 'git -C '+repoPath+ ' log --pretty=format:%H -1'
+                output, errors = shellGitCheckout(cmd, 'latin1')
+
+                line = repoName + ' ' + output + ' ' + str(idx) + '\n'
+                f.write(line)
+                #git diff df055bf81b7aa846530eadb7904d911cf84372c9..df055bf81b7aa846530eadb7904d911cf84372c9 ^
+
+                from dataset import prepareFilesDefects4J
+                prepareFilesDefects4J(repoPath,repoName,output)
+
+                # id = getIssueId('defects/'+pjName+'/'+str(idx)+'.log',pjName)

python/filterBugFixingCommits.py

@@ -0,0 +1,156 @@
+from common.commons import *
+
+ROOT_DIR = os.environ["ROOT_DIR"]
+REPO_PATH = os.environ["REPO_PATH"]
+DATA_PATH = os.environ["DATA_PATH"]
+COMMIT_DFS = os.environ["COMMIT_DFS"]
+BUG_POINT = os.environ["BUG_POINT"]
+COMMIT_FOLDER = os.environ["COMMIT_FOLDER"]
+
+
+
+
+def getLast(bugID):
+    if isfile(join(BUG_POINT, bugID + ".pickle")):
+        return
+    else:
+        subject = bugID.split('-')[0]
+        subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+        repo = subjects.query("Subject == '{0}'".format(subject)).iloc[0].Repo
+        sourceDF = load_zipped_pickle(join(COMMIT_DFS, repo + '.pickle'))
+        sourceDF['fix'] = sourceDF['fix'].apply(lambda x: x.strip() if not x is None else x)
+        aDf = sourceDF[sourceDF.fix == bugID]
+        if len(aDf > 0):
+            dateCheck = aDf.sort_values('commitDate').iloc[0].commitDate
+            filtered = sourceDF.query("commitDate < '{0}'".format(dateCheck ))
+            filtered['dateCheck'] = dateCheck
+            filtered = filtered.head(1)
+            save_zipped_pickle(filtered,join(BUG_POINT,bugID + ".pickle"))
+
+
+def markFix(subject,repoName):
+    subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+    branch = subjects.query("Subject == '{0}'".format(subject))['Branch'].values.tolist()[0]
+    cmd = 'git -C ' + join(REPO_PATH, repoName) + ' checkout -f '+branch
+
+    output, err = shellGitCheckout(cmd,enc='latin1')
+
+    m = re.search(branch, err)
+
+    while not m:
+        time.sleep(10)
+        logging.info('Waiting for checkout')
+
+    aDF = load_zipped_pickle(join(COMMIT_DFS, repoName + '.pickle'))
+    aDF['fix'] = aDF['commit'].apply(lambda x: getBugIds(x, subject, repoName))
+    return aDF
+
+def getBugIds(x,subject,repoName):
+
+
+    cmd = 'git -C ' + REPO_PATH+repoName + ' show --quiet ' + x
+
+    output = shellCallTemplate(cmd)
+    pattern = r"[\s:\[\(#\-/]("+subject+"\-[0-9]+)"
+    match = re.search(pattern, output, re.IGNORECASE)
+    if not match:
+        return None
+    if (len(match.groups()) == 1):
+        matched = match.group(1)
+    else:
+        logging.error('too many match groups')
+    matched
+    return matched
+
+
+
+# def getLasts(subjec,predict = False):
+#     if not os.path.exists(BUG_POINT):
+#         os.mkdir(BUG_POINT)
+#
+#     if predict:
+#         bugIDS = load_zipped_pickle(join(CODE_PATH, subjec+'BugReportsExport.pickle'))
+#         logging.info("Extracting bug points for prediction")
+#         selectedIds = bugIDS.bugID.unique().tolist()
+#     else:
+#         subjects = pd.read_csv(join(CODE_PATH, 'subjects.csv'))
+#         repo = subjects.query("Subject == '{0}'".format(subjec)).iloc[0].Repo
+#
+#         sourceDF = load_zipped_pickle(join(COMMIT_DFS, repo + '.pickle'))
+#
+#         selectedIds = sourceDF.fix.unique().tolist()
+#         selectedIds = [i for i in selectedIds if i is not None]
+#     if subjec != 'ALL':
+#         selectedIds = [i for i in selectedIds if i.startswith(subjec)]
+#
+#     with concurrent.futures.ProcessPoolExecutor() as executor:
+#         try:
+#             futures = {executor.submit(getLast, bugID ): bugID for bugID in selectedIds }
+#             for future in concurrent.futures.as_completed(futures):
+#                 url = futures[future]
+#                 try:
+#                     data = future.result()
+#
+#                 except Exception as exc:
+#                     logging.error('%r generated an exception: %s' % (url, exc))
+#                     raise
+#                 kwargs = {
+#                     'total': len(futures),
+#                     'unit': 'files',
+#                     'unit_scale': True,
+#                     'leave': False
+#                 }
+#                 # Print out the progress as tasks complete
+#                 for f in tqdm(concurrent.futures.as_completed(futures), **kwargs):
+#                     pass
+#         except Exception as e:
+#             logging.error(e)
+#             executor.shutdown()
+#             raise
+
+
+def caseFix(subjec):
+    cmd = 'git config --global diff.renamelimit 0'
+
+    shellCallTemplate(cmd)
+
+
+
+    logging.info("Marking fixes")
+    subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+    if subjec != 'ALL':
+        subjects = subjects.query("Subject == '{0}'".format(subjec))
+
+
+
+
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        try:
+            futures = {
+            executor.submit(markFix, subject, subjects.query("Subject == '{0}'".format(subject)).iloc[0].Repo): subject
+            for subject in subjects.Subject.tolist()}
+            for future in concurrent.futures.as_completed(futures):
+                url = futures[future]
+                try:
+                    data = future.result()
+                    data = data[~data.fix.isna()]
+                    data.fix = data.fix.apply(lambda x: x.strip().upper())
+                    singleFix = data.fix.value_counts().loc[lambda x: x == 1].reset_index(name='count')['index']
+                    singleCommits = data[data.fix.isin(singleFix)]
+                    save_zipped_pickle(singleCommits,join(COMMIT_DFS, subjects.query("Subject == '{0}'".format(url)).iloc[0].Repo + ".pickle"))
+                except Exception as exc:
+                    logging.error('%r generated an exception: %s' % (url, exc))
+                    raise
+                kwargs = {
+                    'total': len(futures),
+                    'unit': 'subject',
+                    'unit_scale': True,
+                    'leave': False
+                }
+                # Print out the progress as tasks complete
+                for f in tqdm(concurrent.futures.as_completed(futures), **kwargs):
+                    pass
+        except Exception as e:
+            # logging.error(e)
+            executor.shutdown()
+            raise

python/fixminer.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+
+source activate redisEnv
+
+PYTHONPATH=$(pwd) python -u main.py -root $(pwd) -job $1

python/getIntroClass.py

@@ -0,0 +1,72 @@
+from common.commons import *
+DATA_PATH = os.environ["DATA_PATH"]
+BUGDIR = join(DATA_PATH,'introclass')
+
+introClassFile = join(DATA_PATH,'introClassData.txt')
+
+def get_filepaths(directory):
+
+    file_paths = []  # List which will store all of the full filepaths.\n,
+    exclude = '.git'
+    # Walk the tree.\n,
+    for root, directories, files in os.walk(directory):
+        directories[:] = [d for d in directories if d not in exclude]
+        java = [i for i in files if not (i.endswith('-diff') or i.endswith('.patch'))]
+
+        for filename in java:
+            # Join the two strings in order to form the full filepath.\n,
+            filepath = os.path.join(root, filename)
+            file_paths.append(filepath)  # Add it to the list.\n,
+
+    return file_paths  # Self-explanatory.\n,
+
+def export():
+    if not os.path.exists(join(BUGDIR)):
+        os.mkdir(join(BUGDIR))
+    # bugList = [i.replace(':', '-').replace('manybugs-', 'squareslab/manybugs:') for i in bugList]
+    # exportCore(bugList[0])
+    bugList = []
+    with open(introClassFile, 'r') as file:
+        for line in file.readlines():
+            bugList.append(line.strip())
+    print("bugList length: {}".format(len(bugList)))
+    for b in bugList:
+        exportCore(b)
+def exportCore(bugName):
+
+    bugNameForDocker = bugName.replace(':', '-').replace('introclass-', 'squareslab/introclass:')
+    cmd = 'docker images '+bugNameForDocker+' --format "{{.ID}}"'
+    logging.info(cmd)
+    output, e = shellGitCheckout(cmd)
+    logging.info(output)
+
+    cmd = 'docker create -ti --name dummy '+output.strip()+' bash'
+    logging.info(cmd)
+    output, e = shellGitCheckout(cmd)
+    logging.info(output)
+
+    if not os.path.exists(join(BUGDIR,bugName)):
+        os.makedirs(join(BUGDIR,bugName,))
+
+    cmd = 'docker cp dummy:/experiment/'+ bugName.split(':')[1] + '.c ' + join(BUGDIR,bugName)
+    logging.info(cmd)
+    output, e = shellGitCheckout(cmd)
+    logging.info(output)
+
+    cmd = 'docker cp dummy:/experiment/oracle.c ' + join(BUGDIR, bugName)
+    logging.info(cmd)
+    output, e = shellGitCheckout(cmd)
+    logging.info(output)
+
+    cmd = 'docker rm -fv dummy'
+    logging.info(cmd)
+    output, e = shellGitCheckout(cmd)
+    logging.info(output)
+
+    homework = join(BUGDIR, bugName) + '/' + bugName.split(':')[1] + '.c'
+    patchName = join(BUGDIR, bugName) + '/oracle.c'
+    cmd = 'diff -u ' + homework + ' ' + ' ' + patchName + '  > ' +patchName +'.patch'
+    logging.info(cmd)
+    output, e = shellGitCheckout(cmd)
+    logging.info(output)
+

python/javaDS.py

@@ -0,0 +1,64 @@
+def createDS(subject):
+    # # if job == 'clone':
+    from commitCollector import *
+
+    caseClone(subject)
+    # # elif job == 'collect':
+    from commitCollector import *
+
+    caseCollect(subject)
+    # # elif job == 'fix':
+    from filterBugFixingCommits import caseFix
+
+    caseFix(subject)
+    #
+    # # elif job =='brDownload':
+    from bugReportDownloader import caseBRDownload
+
+    caseBRDownload(subject)
+    # # elif job =='brParser':
+    from bugReportParser import step1
+
+    step1(subject)
+
+    # elif job =='dataset':
+
+    if not isfile(join(DATA_PATH, 'singleBR.pickle')):
+
+        brs = load_zipped_pickle(join(DATA_PATH, subject + "bugReportsComplete.pickle"))
+
+        subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+
+
+        def getCommit(x):
+            bid, project = x
+
+            subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+            repo = subjects.query("Subject == '{0}'".format(project)).Repo.tolist()[0]
+            commits = load_zipped_pickle(join(DATA_PATH, COMMIT_DFS, repo + '.pickle'))
+            correspondingCommit = commits.query("fix =='{0}'".format(bid)).commit.tolist()
+            if len(correspondingCommit) == 1:
+                return [bid, correspondingCommit[0], project]
+            else:
+                return None
+                print('error')
+
+
+        wl = brs[['bid', 'project']].values.tolist()
+        dataL = parallelRunMerge(getCommit, wl)
+
+        commits = pd.DataFrame(
+            columns=['bid', 'commit', 'project'],
+            data=list(filter(None.__ne__, dataL)))
+
+        save_zipped_pickle(commits, join(DATA_PATH, 'singleBR.pickle'))
+    else:
+        commits = load_zipped_pickle(join(DATA_PATH, 'singleBR.pickle'))
+        subjects = pd.read_csv(join(DATA_PATH, 'subjects.csv'))
+    logging.info('done matching commits')
+    commits['repo'] = commits.project.apply(lambda x: subjects.query("Subject == '{0}'".format(x)).Repo.tolist()[0])
+
+    workList = commits[['commit', 'repo']].values.tolist()
+    from dataset import prepareFiles
+
+    parallelRun(prepareFiles, workList)

python/linuxDataset.py

@@ -0,0 +1,253 @@
+from common.commons import *
+DATA_PATH = os.environ["DATA_PATH"]
+COMMIT_DFS = os.environ["COMMIT_DFS"]
+DATASET_PATH = '/Users/anil.koyuncu/projects/datasets'
+repo = join(DATASET_PATH,'linux')
+
+def filetype_fileter(filename):
+    # return filename.endswith(u'.java') and not bool(re.search('test.*\/', filename))
+    return filename.endswith(u'.c') or filename.endswith(u'.h')
+
+
+
+def checkoutFiles(sha,shaOld, filePath,type, repo=None):
+    try:
+        # folderDiff = join(DATA_PATH, 'gumInput',repoName, 'DiffEntries')
+        folderDiff = join(DATA_PATH, type, 'DiffEntries')
+        folderPrev = join(DATA_PATH, type, 'prevFiles')
+        folderRev = join(DATA_PATH, type, 'revFiles')
+        if not os.path.exists(folderDiff):
+            os.mkdir(folderDiff)
+
+        if not os.path.exists(folderPrev):
+            os.mkdir(folderPrev)
+
+        if not os.path.exists(folderRev):
+            os.mkdir(folderRev)
+
+        # if repo is None:
+        #     repo = join(REPO_PATH,repoName)
+
+
+        savePath = filePath.replace('/','#')
+        if not isfile(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath + '.txt'):
+
+            cmd = 'git -C ' + repo + ' diff -U ' + shaOld + ':' + filePath + '..' + sha + ':' + filePath  # + '> ' + folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt')
+
+            output,errors = shellGitCheckout(cmd,'latin1')
+            if errors:
+                # print(errors)
+                raise FileNotFoundError
+
+            regex = r"@@\s\-\d+,*\d*\s\+\d+,*\d*\s@@ ?(.*\n)*"
+            match = re.search(regex, output)
+            if not match:
+                return
+                # print()
+            not_matched, matched = output[:match.start()], match.group()
+            numberOfHunks = re.findall('@@\s\-\d+,*\d*\s\+\d+,*\d*\s@@', matched)
+            if len(numberOfHunks) == 0:
+                return
+            diffFile = shaOld + '\n' + matched.replace(' @@ ', ' @@\n')
+
+            with open(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath + '.txt',
+                      'w') as writeFile:
+                writeFile.writelines(diffFile)
+
+
+
+            cmd = 'git -C ' + repo + ' show ' + sha + ':' + filePath + '> ' + folderRev + '/' + sha + '_' + shaOld + '_' +savePath
+
+            if errors:
+                # print(errors)
+                raise FileNotFoundError
+            o,errors= shellGitCheckout(cmd,'latin1')
+            cmd = 'git -C ' + repo + ' show ' + shaOld + ':' + filePath + '> ' + folderPrev + '/' + 'prev_'+sha + '_' + shaOld + '_' +savePath
+            if errors:
+                # print(errors)
+                raise FileNotFoundError
+
+            o,errors = shellGitCheckout(cmd,'latin1')
+            if errors:
+                # print(errors)
+                raise FileNotFoundError
+
+    except FileNotFoundError as fnfe:
+        if isfile(folderRev + '/' + sha + '_' + shaOld + '_' +savePath):
+            os.remove(folderRev + '/' + sha + '_' + shaOld + '_' +savePath)
+        if isfile(folderPrev + '/' + 'prev_'+sha + '_' + shaOld + '_' +savePath):
+            os.remove(folderPrev + '/' + 'prev_'+sha + '_' + shaOld + '_' +savePath)
+        if isfile(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt')):
+            os.remove(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt'))
+        # print(fnfe)
+        # raise Exception(fnfe)
+    except Exception as e:
+        # print(e)
+        raise Exception(e)
+
+
+def prepareFiles(t):
+    try:
+        sha,files = t
+
+        shaOld = sha + '^'
+
+        # repo = join(REPO_PATH,repoName)
+        gumInputRepo = join(DATA_PATH, 'gumInputLinux')
+        if not os.path.exists(join(gumInputRepo)):
+            os.makedirs(gumInputRepo)
+
+        # cmd = 'git -C ' + repo + ' diff --name-only ' + shaOld + '..'+sha
+        #
+        # output, errors = shellGitCheckout(cmd, 'latin1')
+        # files = output.strip().split('\n')
+
+        # if len(nonJava) > 0:
+        #     logging.warning('Skipping commit %s',sha)
+        #     return
+        # if len(files) != 1:
+        #     return
+
+        # nonTest = [f for f in files if not re.search('test', f, re.I) and f.endswith('.c') or f.endswith(u'.h')]
+        #
+        # if len(nonTest) > 1:
+        #     return
+
+        nonTest = []
+        for k,v in files.items():
+            if v == 'M':
+                if k.endswith('.c') or k.endswith(u'.h'):
+                   nonTest.append(k)
+        # nonTest = [f for f in files.keys() if f.endswith('.c') or f.endswith(u'.h')]
+
+        cmd = 'git -C ' + repo + ' rev-parse --short=6 ' + shaOld
+
+        output, errors = shellGitCheckout(cmd, 'latin1')
+        shaOld = output.strip()
+
+        cmd = 'git -C ' + repo + ' rev-parse --short=6 ' + sha
+        output, errors = shellGitCheckout(cmd, 'latin1')
+        sha = output.strip()
+
+        if isinstance(nonTest, list):
+            for file in nonTest:
+                checkoutFiles(sha,shaOld,  file,'gumInputLinux',repo)
+
+
+
+    except Exception as e:
+        print(e)
+
+
+def checkCommitLog(x):
+
+    cmd= 'git -C ' + repo + ' show ' + x + " --pretty=\"format:\" --name-status -M100%"
+
+    out, err = shellGitCheckout(cmd, enc='latin1')
+    log = {}
+    lines = out.strip().split('\n')
+    for line in lines:
+        fname = line[2:].strip()
+        ftype = line[:1]
+        log[fname] = ftype
+    log
+    return log
+
+def getCommitLog(x):
+
+    # commit, repo = x
+
+    cmd = 'git -C ' + repo + '/ ' + " show --pretty=format:'%B' --no-patch " + x
+
+    output = shellCallTemplate(cmd, 'latin-1')
+
+    matches = re.finditer('Fixes: [0-9a-f]{6,40}', output)
+    match = list(matches)
+    fixes = []
+    if len(match) >= 1:
+        for m in match:
+            fixes.append(m.group())
+    #     return match[0].group()
+
+    matches = re.finditer('http[s]?:\/\/.*show_bug\.cgi\?id=[0-9]*',output)
+    match = list(matches)
+    links = []
+    if len(match) >= 1:
+        for m in match:
+            links.append(m.group())
+
+    df = pd.DataFrame(data=[[links,fixes, output,x]], columns=['links','fixes','log','commit'])
+    # df = df.T
+    # df.columns = ['log', 'commit']
+
+    return df
+
+
+
+    output
+
+def collectBugFixPatches():
+    commits = getAllCommits()
+    # remove commits that are only deleting or adding files
+    commits = commits[commits.files.apply(lambda x: np.any([i == 'M' for i in x.values()]))]
+    # keep only commits that are changing c files (.c)
+    commits = commits[commits.files.apply(lambda x: np.all([i.endswith('.c') for i in x.keys()]))]
+    #not a revert commit
+    # commits = commits[~commits.log.apply(lambda x: x.startswith('Revert'))]
+    # commits = commits[commits.files.apply(lambda x: len(x) == 1)]
+    # commits['cocci'] = commits.log.apply(lambda x: True if re.search('cocci|coccinelle', x) else False)
+    # coccis = commits[commits.cocci].commit.values.tolist()
+    fixes = commits[commits.fixes.str.len()!=0].commit.values.tolist()
+    # links = commits[commits.links.str.len()!=0].commit.values.tolist()
+
+    # bugs = set(fixes).union(links).union(coccis)
+    # bugs = set(fixes).union(coccis)
+    bugs = set(fixes)
+    a = commits[commits.commit.isin(bugs)]
+    # for s in a.commit.values.tolist():
+    print(len(a))
+    parallelRun(prepareFiles,a[['commit','files']].values.tolist())
+        # prepareFiles(s)
+
+
+def markBugFixingPatches(commits):
+    # from pandarallel import pandarallel
+    #
+    # pandarallel.initialize()
+    # commits['files'] = commits.commit.parallel_apply(checkCommitLog)
+    # commits
+
+    # commits['isC'] = commits.files.apply(lambda x:np.any([i.endswith('.c') or i.endswith('.h') for i in x.keys() ]))
+
+    commits = commits[commits.isC == True]
+
+    # commits.commit.parallel_apply(getCommitLog)
+    f = parallelRunMergeNew(getCommitLog, commits['commit'].values.tolist())
+
+    res = pd.merge(commits, f, on=['commit'])
+
+    save_zipped_pickle(res, join(COMMIT_DFS, 'linuxFix' + ".pickle"))
+
+
+def getAllCommits():
+    if isfile(join(COMMIT_DFS,'linuxFix.pickle')):
+        return load_zipped_pickle(join(COMMIT_DFS,'linuxFix.pickle'))
+    else:
+
+        if isfile(join(COMMIT_DFS,'linux.pickle')):
+            commits =  load_zipped_pickle(join(COMMIT_DFS,'linux.pickle'))
+        else:
+            if not os.path.exists(COMMIT_DFS):
+                os.mkdir(COMMIT_DFS)
+
+
+            cmd = 'git -C ' + repo + " log --no-merges --pretty=format:'{\"commit\":\"%H\",\"commitDate\":\"%ci\",\"title\":\"%f\",\"committer\":\"%ce\"}' > " + join(DATA_PATH,'linux' + '.commits')
+            output = shellCallTemplate(cmd, enc='latin1')
+
+            from commitCollector import makeDF
+            rDF = makeDF(join(DATA_PATH,'linux' + '.commits'))
+            save_zipped_pickle(rDF, join(COMMIT_DFS, 'linux' + ".pickle"))
+            return rDF
+        markBugFixingPatches(commits)
+
+

python/main.py

@@ -0,0 +1,235 @@
+from common.commons import *
+
+
+
+
+
+if __name__ == '__main__':
+
+
+    try:
+        args = getRun()
+        setLogg()
+
+
+        setEnv(args)
+
+        job = args.job
+        job = 'indexClusters'
+        ROOT_DIR = os.environ["ROOT_DIR"]
+        REPO_PATH = os.environ["REPO_PATH"]
+        CODE_PATH = os.environ["CODE_PATH"]
+        DATA_PATH = os.environ["DATA_PATH"]
+        COMMIT_DFS = os.environ["COMMIT_DFS"]
+        BUG_POINT = os.environ["BUG_POINT"]
+        COMMIT_FOLDER = os.environ["COMMIT_FOLDER"]
+        FEATURE_DIR = os.environ["FEATURE_DIR"]
+        DATASET_DIR = os.environ["DATASET_DIR"]
+        PROJECT_TYPE = os.environ["PROJECT_TYPE"]
+        jdk8 = os.environ["JDK8"]
+        pd.options.mode.chained_assignment = None
+
+
+        subject = 'ALL'
+        rootType = 'if'
+
+        print(job)
+
+
+        if job == 'dataset':
+            from javaDS import createDS
+            createDS(subject)
+        elif job == 'linuxDS':
+            from linuxDataset import collectBugFixPatches
+            collectBugFixPatches()
+        elif job =='otherDS':
+            from otherDatasets import core
+            core()
+        elif job =='richEditScript':
+            cmd = "JAVA_HOME='" + jdk8 + "' java -jar " + join(DATA_PATH, 'FixPatternMiner-1.0.1.jar') + " " + join(DATA_PATH, 'app.properties') + " RICHEDITSCRIPT " + 'L1'
+            output = shellCallTemplate(cmd)
+            logging.info(output)
+
+        elif job =='loadRES':
+            cmd = "JAVA_HOME='" + jdk8 + "' java -jar " + join(DATA_PATH, 'FixPatternMiner-1.0.1.jar') + " " + join(DATA_PATH, 'app.properties') + " LOAD " + rootType
+            output = shellCallTemplate(cmd)
+            logging.info(output)
+
+        elif job =='shapeSI':
+            from pairs import shapePairs
+            matches = shapePairs()
+
+            from pairs import createPairs
+            createPairs(matches)
+
+            from pairs import importShape
+            importShape()
+
+        elif job =='compareShapes':
+             # cmd = "mvn exec:java -f '/data/fixminer_source/' -Dexec.mainClass='edu.lu.uni.serval.fixminer.akka.compare.CompareTrees' -Dexec.args='"+ " shape " + join(DATA_PATH,"redis") +" ALLdumps-gumInput.rdb " + "clusterl0-gumInputALL.rdb /data/fixminer-core/python/data/richEditScript'"
+            cmd = "JAVA_HOME='" + jdk8 + "' java  -jar " + join(DATA_PATH, 'FixPatternMiner-1.0.1.jar') + " " + join(DATA_PATH, 'app.properties') + " COMPARE " + 'L1'
+            output = shellCallTemplate(cmd)
+            logging.info(output)
+
+        elif job == 'clusterAdditional':
+            from addNewData import cluster
+            cluster()
+
+        elif job == 'cluster':
+            from abstractPatch import cluster
+
+            dbDir = join(DATA_PATH, 'redis')
+            startDB(dbDir, "6399", PROJECT_TYPE)
+            cluster(join(DATA_PATH,'shapes'),join(DATA_PATH, 'pairs'),'shapes',rootType)
+
+        elif job =='actionSI':
+            from pairs import actionPairs
+            actionPairs(rootType)
+
+        # elif job =='importActionPairs':
+            from pairs import importAction
+            importAction(rootType)
+
+        elif job =='compareActions':
+            # cmd = "JAVA_HOME='"+jdk8+"' java -Xmx8096m -Djava.util.concurrent.ForkJoinPool.common.parallelism=64 -jar "+  join(DATA_PATH,'CompareTrees.jar') + " action " + join(DATA_PATH,"redis") +" ALLdumps-gumInput.rdb " + "clusterl1-gumInputALL.rdb"
+
+            cmd = "JAVA_HOME='" + jdk8 + "' java -jar " + join(DATA_PATH, 'FixPatternMiner-1.0.1.jar') + " " + join(DATA_PATH, 'app.properties') + " COMPARE " + 'L2'
+            output = shellCallTemplate(cmd)
+            logging.info(output)
+
+        elif job == 'clusterActions':
+            from abstractPatch import cluster
+
+            dbDir = join(DATA_PATH, 'redis')
+            startDB(dbDir, "6399", PROJECT_TYPE)
+            cluster( join(DATA_PATH, 'actions'),join(DATA_PATH, 'pairsAction'),'actions',rootType)
+
+        elif job == 'tokenSI':
+            from pairs import tokenPairs
+            tokenPairs()
+            from pairs import importToken
+            importToken()
+
+        elif job == 'compareTokens':
+            # cmd = "JAVA_HOME='"+jdk8+"' java -jar "+  join(DATA_PATH,'CompareTrees.jar') + " token " + join(DATA_PATH,"redis") +" ALLdumps-gumInput.rdb " + "clusterl2-gumInputALL.rdb"
+            cmd = "JAVA_HOME='" + jdk8 + "' java -jar " + join(DATA_PATH, 'FixPatternMiner-1.0.1.jar') + " " + join(DATA_PATH, 'app.properties') + " COMPARE " + 'L3'
+            output = shellCallTemplate(cmd)
+            logging.info(output)
+
+        elif job == 'clusterTokens':
+            from abstractPatch import cluster
+
+            dbDir = join(DATA_PATH, 'redis')
+            startDB(dbDir, "6399", PROJECT_TYPE)
+            startDB(dbDir, "6380", "clusterl2-gumInputALL.rdb")
+            cluster(join(DATA_PATH, 'tokens'), join(DATA_PATH, 'pairsToken'),'tokens')
+            stopDB(dbDir, "6380", "clusterl2-gumInputALL.rdb")
+
+        elif job == 'additional':
+            from addNewData import core
+            core()
+            # from addNewData import checkWrongMembers
+            # checkWrongMembers()
+
+        elif job == 'codeflaws':
+            from otherDatasets import codeflaws
+            codeflaws()
+
+
+        elif job =='indexClusters':
+            # from sprinferIndex import runSpinfer
+            # runSpinfer()
+            #
+            # from sprinferIndex import test
+            # test()
+            # from sprinferIndex import divideCoccis
+            # divideCoccis()
+            # from sprinferIndex import removeDuplicates
+            # removeDuplicates()
+
+
+            from patchManyBugs import patchCore
+            patchCore()
+            # from patchManyBugs import patched
+            # patched()
+            from patchManyBugs import exportSosPatches
+            exportSosPatches()
+            from validate_manybugs import validate
+
+            validate()
+
+        elif job =='patternOperations':
+            from sprinferIndex import patternOperations
+            patternOperations()
+        elif job == 'patchManyBugs':
+            from patchManyBugs import buildAll
+            buildAll()
+
+            # from patchManyBugs import patchCore
+            # patchCore()
+            # # from patch_validate import patch_validate_mine
+            # # patch_validate_mine()
+            # from patchManyBugs import patched
+            # patched()
+            # from patchManyBugs import exportSosPatches
+            # exportSosPatches()
+
+        elif job =='patchIntro':
+            from sprinferIndex import patchCoreIntro
+            patchCoreIntro()
+            # from sprinferIndex import patched
+            # patched()
+
+        elif job =='validateIntro':
+            # from patch_validate_introClass2 import patch_validate
+            # patch_validate()
+            from test_patched_file import patch_validate
+            patch_validate()
+        elif job =='checkCorrectIntro':
+            from test_patched_file import checkCorrect
+            checkCorrect()
+        elif job == 'manybugs':
+            from getManybugs import export
+            export()
+
+        elif job == 'validateMany':
+            from patch_validate import patch_validate
+            patch_validate()
+
+        elif job == 'introclass':
+            from getIntroClass import export
+            export()
+
+        elif job =='stats':
+            from stats import statsNormal
+            statsNormal(True)
+
+
+        elif job == 'datasetDefects4J':
+            from defects4JDataset import core
+            core()
+
+        elif job =='bug':
+            from bugstats import bStats
+            bStats()
+        elif job == 'defects4j':
+            from stats import defects4jStats
+            defects4jStats()
+        elif job == 'exportPatterns':
+            from stats import exportAbstractPatterns
+            exportAbstractPatterns()
+        elif job =='export':
+            patternPath = join(DATA_PATH,'actions','ExpressionStatement','3','0','0')
+            patterns = listdir(patternPath)
+            for pattern in patterns:
+                repo = pattern.split('_')[0]
+                file = pattern.replace(repo+'_','')
+                print(file)
+                filename = file.rsplit('_',1)[0]
+                print(join(DATA_PATH,'gumInput',repo,'DiffEntries',filename))
+                break
+
+        else:
+            logging.error('Unknown job %s',job)
+    except Exception as e:
+        logging.error(e)

python/mapping.csv

@@ -0,0 +1,395 @@
+commons-math eda09ddbf0722f2a381937714c684669869641dd 1
+commons-math 0b10c69d0e684d4e6585183c9d67c8b00ce0abe0 2
+commons-math 04e9b9ff0b35a20dfdbaf57a49b0981b4d3a715e 3
+commons-math 94082227b21dddc8a9fae78ace5de379889295f2 4
+commons-math 9b8f396da2cb7fa552bfed3b35505f4994f02ceb 5
+commons-math 089aaa8a9ea362245a8e1060c22cd0c1ee167e9d 6
+commons-math 2c0b62235cc52f4e4991c04aa40d27caaa999359 7
+commons-math 1b6d8693035b7b5fff0cffa935efd9763a38a547 8
+commons-math 973751d8070e87d58870e60dd737fdf01ee101ad 9
+commons-math 76768bea8a4ad5dc4391022bb11e6ed83a703825 10
+commons-math 22d4700dbf1938f3a12063ea5e1d667b8bc3d0e1 11
+commons-math 6f67acee3ebaac338796ea6d0fa121294c658d84 12
+commons-math 71972b527b9a001db1883d40612a979b0e56fbf3 13
+commons-math b69cf971c662c8f289c3c26f8a234fef5891345c 14
+commons-math 387b09d317e0b9c705798e0a398e7869fd388478 15
+commons-math 2195f51ffa81b2a2e66aa23ea1565ec260dbc2c4 16
+commons-math f0a4cd94185da3dbaea419e363403d6f279ba7ff 17
+commons-math f566bb56928c3eec5d3b7dd306b994589bb60b69 18
+commons-math 33e517f17b2fda199e284a10248aae8987fdc7c8 19
+commons-math cc07b85139799c361f84ccfb587f25e7b9089758 20
+commons-math ca94a1d1aba5c858522e61c8e39e61d147969ade 21
+commons-math 338928edd75bffbe2017cbf7e70e7eb47b5aeef8 22
+commons-math 7762063c11777f6e4bc328b872f28dd50ebb47d5 23
+commons-math efd381b8e7f807af2554589e1e5aa0823d80ba40 24
+commons-math 003e5dc9d09b7c654d3ca4a826b7f773dcac9e68 25
+commons-math 197125f3002bfa00116136c9ae4f473aca9d9e50 26
+commons-math 3be7c4e930206cf953dcf0f6b67bfb2cf89120fc 27
+commons-math ddd0e0547a8f49bf524bc8cbce7d4db5de08fe28 28
+commons-math d1cfc57c254cc8961763c97d24b1332f6301fd8f 29
+commons-math 695d4f2d86c166903ac91662e1510e7a9274cfe6 30
+commons-math ac85e774762e1eb74bdc7ca32252cd8b45836127 31
+commons-math 5c1f8091eeeac692c67d1abb433cdf25ef094b77 32
+commons-math 8c4756dadf1940b8aeb8bf0086ef0be53369073e 33
+commons-math abecdd89026cdd8dc6b23b4a4444ac490df4b86c 34
+commons-math b09e77f7a4a6941d477a761fafa2cfe71957ccd0 35
+commons-math a9fdc0b87edc8c05dba67b6393daa115e9c7678c 36
+commons-math 925281238fa5d5f8eb210c12447500492b0e87d9 37
+commons-math 7b4260d6c392e163fd3809129ea5e3fca9a265ea 38
+commons-math 94e79f91037f4e0a1502d8544a74e215920c41ee 39
+commons-math d7c2c4660f1d286efcbc111bfc0b20d009758edd 40
+commons-math 63883fa3aaf7f7e233668be16697815d5db87eec 41
+commons-math 19f75238b4782f2fce756fd9e86682ee77941a88 42
+commons-math b6ca86d27583b098f06459fe25c5cf4783dec7d1 43
+commons-math 58b4fb481e15f0e4843381a2fa604a707e1ce48e 44
+commons-math f1b968888f0e67cd1d92f7a40a9b1e662ccde108 45
+commons-math 7e791a884b85418e5ec071de9b30bbee0adcdada 46
+commons-math 0fbe523e741cb387f3fd71cb268f8d6abc96808e 47
+commons-math 93589c33933453cafccfd65217b693ac9478c62c 48
+commons-math 5a7e1a1c4f6b2a940fcd0ec5a02d543eddde9231 49
+commons-math 78f340e2782463533f9735e6ea0a780370dc44db 50
+commons-math ef2b4d4935930601ff2c1a140e0bff90b21b564e 51
+commons-math a77e71654ac1abc83788244f331540d54ecd8c90 52
+commons-math dfbd011509655d4b4f5c4f1c94a533f1843e693c 53
+commons-math 902a19844b47b977edd2ffeaac67c6196c464026 54
+commons-math 429e2bfc4af25cd217cfaab9600679a9bb57b124 55
+commons-math 768aef92d8c6470714b7ae43fb17cf171bbbbd89 56
+commons-math 309d8b0ffd17cdf2d8d971e76dd0ea9011b87019 57
+commons-math 237b424e0b73bc7b3e66a103937abdbc61055d06 58
+commons-math ac9d07d8071062b9084730e885ec8672698bf3c8 59
+commons-math d8929eb622983ab1c69c3c9b0ffe4df3e93d798c 60
+commons-math 60be2e3c9af6e8b084fc88ad51cfd36216dd566e 61
+commons-math 5fcb6b272159abf421a4eb0b8c1cab052d9b663e 62
+commons-math abd5e6b3fd30a83795b83f60fe4074781327e773 63
+commons-math 35114e428c9586de69a2a5a8977a340c115b5c6b 64
+commons-math 9139bb81e78b7781d5656dcf08e57596115e29a1 65
+commons-math 83f4643ef0718dfc1146861f99859495fa9f7022 66
+commons-math 78d9cae51fe6c541440d45c5db1fff39b67e2aaf 67
+commons-math 43892aeddef53ae33f7a7e28576fb1c86af040cb 68
+commons-math 6ce0161426530b434c8bb19bc54cfd662374ff88 69
+commons-math af488ee221c5cac50e3d3084b657878b33101c58 70
+commons-math 20fcc9659c91ec24dad698058f727d245b79cf19 71
+commons-math 5511a4caec50ae8de127a5d37ad702897a2b437e 72
+commons-math a2a5ba56b5423f217540fac031b99c40771d3c5c 73
+commons-math c713dcd255634afcc07010c4ebbc9222837beda1 74
+commons-math 08b7f78245c006f1db5ba8f70376d4ed3f95fd1f 75
+commons-math 0f6f076d348dea47de4edc22224e7424a64787e0 76
+commons-math 0fe9219a05f5f5951279b726cb370ba12daab9d1 77
+commons-math 1bdaff00971703812f9792878f3872e1f582c712 78
+commons-math e41aac17c1576ecf66b16c3dcf3a321618e45ce8 79
+commons-math 8f53ff33b4114441d4c6c60d8c77de1748885346 80
+commons-math 08aeef3003a0ca9753a43ee2ad1c3e6e54a5074e 81
+commons-math 9375b5b78d4f40e644febf356394f914dbdd3680 82
+commons-math 0de2b4f6a02d3d8a5f20d2f3eacfcb4518031ce0 83
+commons-math 486a9058c65799753f137435ff14264ded3018d9 84
+commons-math 505277def255d62e673975bce0c04ce710c9d790 85
+commons-math 42db931b48273cca05423768e08fa159aa8923cb 86
+commons-math 400bc3b17626564cf11d0a33ff08eae718511289 87
+commons-math dd13f84f87c244ee0db51e7805c63e47ac1b6cbe 88
+commons-math 8cd39500a39e942a024aa5af53d43886f85238df 89
+commons-math f5b79274eabc702bdd89f49e6347aef56a8ce761 90
+commons-math 54d471e629ec3a1ccf2ef88fcceae677efcfe74d 91
+commons-math 3960af79a6305bbc8f4de33e4ca9224a230f74e0 92
+commons-math 96de20bca7f370a08c35ec70ddf851954a6b5e0f 93
+commons-math 455f53c2d6f5855a7e38122615ecd3092eecad8d 94
+commons-math 086729a4f385f3ea9a510e60cb7e8d6811616dfb 95
+commons-math 3e6fcb88415c7a6f2b3254ef9932f986f338b80e 96
+commons-math d070557636216ce26a3fafc080a163eac7ae5db1 97
+commons-math f2118d42cedb487bb56477066131ec9a3421b1e8 98
+commons-math e6a07fd07a586bd92a1b4568c85278a666731995 99
+commons-math 1734f3e722b694c0d60480cc04bbfdedd0ab356f 100
+commons-math d86f804271b77640aac55773c8da9756653f1869 101
+commons-math 7b1b7cf2b9cde663e8956bfcfccbc6367819e74d 102
+commons-math 6b020c234146df3b2bbba4f636805cdc5d4d55bf 103
+commons-math 91697fd97b85655bc7f54be1e32ae73a10fcef87 104
+commons-math 96b644f7e0a6da836f4fa9e3cfe9d68c2d155bd0 105
+commons-math 682c35c46895ab19396c37f402ebdaf396c5b33a 106
+commons-lang 895e948784d33ee8c77447faaec7a31d7ac59e67 1
+commons-lang be468f734d6415f0586f00ae80078b6c2db20f5c 2
+commons-lang d32ea50b546af2a6d5706ae076fce062f9a8c8f0 3
+commons-lang 930ed0fcc7f753dc37088ae87d928ede99755af2 4
+commons-lang e56f3df5b4490fa6a899ff09975bf0a65edf9bf4 5
+commons-lang 25377515553ef487734795dd01e2a0fe53d81cb7 6
+commons-lang e24f94666a1da3e1d096d7c1c5e12b8b6af30a07 7
+commons-lang 7ce1b25e54626176a8ac754955305810d5b2a0ba 8
+commons-lang bebe6d022e5c025b43e9d85526142a843e5c0efc 9
+commons-lang 6db4df1116ac19e9ede52077d3ac8877f1f2fa76 10
+commons-lang 8875ff45172c32bc31dab5ce4f552df2fdef99ad 11
+commons-lang de56ca5a30bb370980eb58f25846cd8ce385414c 12
+commons-lang 59eed5927bcb00bafd79835a7975778443cf9373 13
+commons-lang 40e8be5be33d9f87fd7f41d68b058ba9f1fee241 14
+commons-lang 0eaf4d2c5a210dc634a500a7b911610e969f636b 15
+commons-lang 4cc8649da1f56cb985cbf38eeb3befac26da3052 16
+commons-lang 0c0aed5b22f872a26a3c4b3470eb4846a5c6a510 17
+commons-lang 2c85a6997e40fc38e4eb7eb7c4b17e10e880260a 18
+commons-lang 63b4cd6d5b0bfa27ff4192af47be021daa24360e 19
+commons-lang ccd0932c14a5634b0632950feffbd464aa5f11dc 20
+commons-lang a6d82be18325519dbc0b17f473cfb2523e440025 21
+commons-lang 294ef3c222222f34ace3cf31a1a686d23279a91e 22
+commons-lang 5cfa022910c76d52a4335c5fde48d9d588571e70 23
+commons-lang e973d28686fffd1a7b5af13010be661f42709049 24
+commons-lang 93c5f342813c9415d3fa6d1859b8afd7cd22cea2 25
+commons-lang 49fcd95ddb86d18c3693501b7ab77e6f668b2738 26
+commons-lang c991609ffb2daa0d11e3ed7dd647f29734dc09e3 27
+commons-lang 018eec3ac2e4366a073ea54f62f9ff225301071d 28
+commons-lang 9827edce09f17dd6c7ccc36f1bb3d635dbbf8d0d 29
+commons-lang 0cfd84bac2c5ff8937f5bfc52ccef30b9b208764 30
+commons-lang 5a1af029d3df5ea2868f50c51d365025076a788e 31
+commons-lang 67e6482466abf70c9f65ecd53277c316770e240b 32
+commons-lang ea38f61b485079c957021d7088b1e04b96cb822d 33
+commons-lang c4718335a6ab4826858af76197baf629eca9bbaa 34
+commons-lang 855529a649c8f10d72aa3860f4713eea73ad0f3e 35
+commons-lang 3a7a2c95c8cda037467aa61458b74e9c5a6bfddf 36
+commons-lang d6ac2816d909a5283a04205e472228ef38ee892f 37
+commons-lang 7f1fb35c67a4d33309551ac99cb09581549c6e4f 38
+commons-lang 49e2897613d6aa6b9ab7d81383a744b143e26105 39
+commons-lang a7c71fd3dc2b78aabda01065b7cbc6be8224f3e7 40
+commons-lang de8a1dd8d94f99df5fba033a53c7c2c6a359093b 41
+commons-lang 7a01e79423e8583fe085c18f687fc2d76a47ba99 42
+commons-lang 648e09dd6fa7c1f5b8ceee92e8801a2331bc9700 43
+commons-lang 8fb0fc2c0efff7bd580ad58c8366150ddf2ec9a0 44
+commons-lang 5427389a0011f16a8d8f795d4280dde25c0a19a1 45
+commons-lang 573dc2c1ebbb077377f4ee6339b6a3089112a743 46
+commons-lang 6a56384d0a21320e7cee4d313904cdbf8b5568a5 47
+commons-lang f6fa9920f3005334bb8f9ff9d37d0b3216b2e0c8 48
+commons-lang 4a93191004e50a98c483c5f9a2c3212cea980e62 49
+commons-lang adde748b048cadbedff7faffdc3fa47be6602a1b 50
+commons-lang d8e8231499756d5b5f7a15d4fe88b62abca28a4c 51
+commons-lang d1c1b234ff840616585cfb85aa39975138c877d5 52
+commons-lang fd1d5c8d59df0bc5e14ad377d2a11723bc09f3ae 53
+commons-lang 00adee4181d841800e2d9259cd0e899a08dbf895 54
+commons-lang dcf847b42cee091f8aca281fdc71a064d81af7ce 55
+commons-lang c44c9aa6c17eadc6e0dc1786791354c6d2e9a72f 56
+commons-lang fc67e5eb058d973fbde2adf314b16949301d96ac 57
+commons-lang b398d36a2f8609fa9f7a8b648635e7103dfaa075 58
+commons-lang c11e56cd5fd574bc78ec1526f7f6ff4532e4107d 59
+commons-lang c7007240825f14068fd3de5bd4eb02213763404b 60
+commons-lang facc2891634d802effa316d0b8681c662121074a 61
+commons-lang ce597858216844c3e6b48e6f57876a5e7da68843 62
+commons-lang c5284738904a14738de6981fddb92f26ccfa5081 63
+commons-lang f3a0047b6e7ebe683a8e36bb472cc959c53b0a15 64
+commons-lang 2a201a9820fccc8f246c082449a6d70919d8b2d3 65
+closure-compiler 17b42649be30670a6451793b0e52270332f30c5e 1
+closure-compiler 128fe40e61b1b4e1b9fcff829469b4c7a5c6a0e0 2
+closure-compiler 3f61a514576def868e65748faf148d565c4df9a9 3
+closure-compiler 4242311353fb347aac55b39300281621850a3e94 4
+closure-compiler ab2d9b28ff1cc44e18ad86057019ecee109969e8 5
+closure-compiler e543c554a0fc1d8ed26c9e92d6c1c48dc20a3fb8 6
+closure-compiler f5abfa559006cb8e382961783a8f9c449a306573 7
+closure-compiler 97cc094dd2fa6db9897efe73a399ae3c9fe7f8ae 8
+closure-compiler 5fb451ab2754c7326af51cb095fe3ee3f40793c9 9
+closure-compiler 64df8a81cee31175a7782238258676380ccdec99 10
+closure-compiler c1fd2cf2d9e1f62599b6ba113265dd71835bc58e 11
+closure-compiler 6b7bf3ce8a397138a11ff1c247128899532a5d71 12
+closure-compiler d20b1e22b228cf68f51ffa96bfa3fd29779a04fd 13
+closure-compiler d287f14f8f414f3bdafdcc136ac739ff20071ef4 14
+closure-compiler 2158b84e4ba237206abe8e8b742e4ccac9710a47 15
+closure-compiler 0e3d7a045fb69db9e1082c7f33527df9216ee37e 16
+closure-compiler c0ba69fe6a68b102791aa20442dee43cc2b394a0 17
+closure-compiler 7976de9f51dc6461b6d963f3b6ac5739a1329f8e 18
+closure-compiler 949261ba98c74ff2fc05e19782e1524c775a286d 19
+closure-compiler a8bafc090a63c9a6cb15fda3618e3d503201ff62 20
+closure-compiler 8d515414a135e3f13583113a4a48a9139f90443f 21
+closure-compiler 73894a1bcb85c7499635a185f076ee3a20e58655 22
+closure-compiler 7352103c926bf8fb44a3b31ffdf1641768457297 23
+closure-compiler 194eef59efccb9b88de33654c5b2d5250775e86d 24
+closure-compiler 09c465f4b1de939e8d42f000a57f153233618830 25
+closure-compiler c66b1ba06eb364f0f703175efac1939713130b01 26
+closure-compiler 8534b64004d3dbf35184a46a4644b8854a5953d9 27
+closure-compiler 723221c8505b276220e5d46edafbe872cdd9448e 28
+closure-compiler 0604636aeb78f981535cb61e03c7caf14324f480 29
+closure-compiler 88eccac44fcff18012747103c79148df8fafd354 30
+closure-compiler 289a407dcd572ec7bebb968d2c1f33dd34013c6b 31
+closure-compiler ac6bdd635ce4f1e855c4dc32adfeedea2a3c65a3 32
+closure-compiler 092dd0740b185fd7e692c75fc4e7fdfb3a34a587 33
+closure-compiler b46f537e3e2b1a71c11760a8fccb582fc9d9d9bc 34
+closure-compiler 2248c10356b7b2102fe1f800c15a24c46c3a36a6 35
+closure-compiler 8a3c48a8e947ddf7cc478379c89d839a054aa2fd 36
+closure-compiler 81c5170de94d8aab7cbc22a5ea70ffb05d47b970 37
+closure-compiler 33bfccb4f43dab67fc18fe896e7a53b0840ba638 38
+closure-compiler 1536e1f36db69cd16156798b113af21acac31c80 39
+closure-compiler 7ce65ef6c2f41cc0ab0ef443f04f481188aa4390 40
+closure-compiler 6c4cc204da52f791659fd74bb42330606f44758a 41
+closure-compiler 46b4fd226132b2eb8c10c867a57ea7f3b02fea22 42
+closure-compiler 67a0910362f36715edb08219375b3171e8e4ed25 43
+closure-compiler 6d4bcf0801193a78f47f9c535b3e7013d1d8a04b 44
+closure-compiler 8467e09e27886a6958038741c40676438f0cbcbf 45
+closure-compiler c9522a14d06c9ca56b042cf56285a675dc62b465 46
+closure-compiler 39415d1336cfb194550bc1de03f767f4e9594e3f 47
+closure-compiler 22082d48b4e9af1bc206eb817011da433752d0e9 48
+closure-compiler cff75ea4232d14ce826d01aca16313927c776159 49
+closure-compiler 5fbc32c893da4d526aa2e1ad8100dcf5038463d7 50
+closure-compiler 90b1b911653f6ed8f633efc7a73b137fdd1ba4e0 51
+closure-compiler caf9b7026ffc4c46b65d784d1ae33d72cba4df7e 52
+closure-compiler 8ae4485d3f29133e07692b91b9e26e3779e36745 53
+closure-compiler 9be8fc324a48a61ff264f51c95e1845c26915ba9 54
+closure-compiler eb5c918437cd3b26df1dd6ed8117c7a9dc63ed35 55
+closure-compiler 050f47ca22c049c3b33f124280b6994d06d5629c 56
+closure-compiler 7c95052af3d4eaac0fd26bbf78de0ec694a34f3d 57
+closure-compiler 53ed8cc8d8e5eec7d70a99457cf9fefe07dece5a 58
+closure-compiler 56ce187569b2dc22dd80c9b26c5e504b9ed12a48 59
+closure-compiler 38181744e86aaf54781929c26d062b157637ed9d 60
+closure-compiler 8b6be42415b3f42e981764a4b9a35f63c7348b9d 61
+closure-compiler 3de13d3c9defe54035fd31884b1fd1ac4454cdf1 62
+closure-compiler 214c9d5d4ab5223e9bd27ff6ff57f4b009fe7a3c 63
+closure-compiler f65825ffb7944f19afb947bcecb4997e56707f59 64
+closure-compiler 7a75c11f0697a1631b2a04bcce3e6a338bfc583d 65
+closure-compiler 48eee6a5de219e9201f2b62f770c03c2811193d7 66
+closure-compiler b648f1975bfb45282ed596ab912b142e8b8bda1b 67
+closure-compiler ba0c627fa4724787b333a116d7eaac913f06dfbb 68
+closure-compiler ba21d4bf52d37f5dff7915b6e48a5cc236303276 69
+closure-compiler 12ba45a5ad0cb089cffd4f410833e1c49c940bd1 70
+closure-compiler c0e583acbaa177467f117aee208e2b38f4b40d3d 71
+closure-compiler 2dc78788ef877c7c71584b2576676ce4524b8a43 72
+closure-compiler ae75590caa91a52ab9ea78285313ab374d8ce9c7 73
+closure-compiler c5059efe668cdd7ec81cfde843c92371c0f5c8f8 74
+closure-compiler dc609190c02bda906b8048b5d35b0efd7e1a98d1 75
+closure-compiler ab3afccd1a57b5b414fd1c200eea7ebab26ebd83 76
+closure-compiler 922201e1e17b941052900f8412635adef01455f7 77
+closure-compiler 633276ea84510c6c5ff7f3d0527dcab37aac376f 78
+closure-compiler c4b8c1e3e36e06fd469daf0336e24d7d5a391894 79
+closure-compiler 1d28cfba615168eca30d9d854a44201a0d6585f5 80
+closure-compiler 45083a92ededbcefe9055ec89b8e759bf2ab452b 81
+closure-compiler 777385613cc250bfa295d7d2b301577e427c99d4 82
+closure-compiler dc770dd2dcb1b0a6ff1d9e051cc14b10cb440fbf 83
+closure-compiler 1324da7353cb3075804f8864d2a8d1acdeaa4cbb 84
+closure-compiler 770af0422fdbeb8c932a840fae3df4d6233da9ca 85
+closure-compiler 4dd5bb97cbca4aa42d0a77e24855ab32e3687587 86
+closure-compiler 49f20b87829889bdd826dc83551d1e3ba63fd9dd 87
+closure-compiler 5f2f5f714951f2c5ae16c50b3359d5665695395d 88
+closure-compiler 158a8ce92a5cc755b2125a35e91beb481fc0e360 89
+closure-compiler 7f443533895b084a0c2e9e6eeba2c30d1add6809 90
+closure-compiler 5e9b9bd8d056f2fea6ad0a8107d029bd5cb14aab 91
+closure-compiler 1df9ddf552b80a7970b7396ec1e859f31a2a652c 92
+closure-compiler af76e1088a84a2af3932178d138d439320f53600 93
+closure-compiler d537e62782e605ef50b06eae38103ebe1fff08be 94
+closure-compiler 446fca32512b954ba38fe59fb16e129f3e096cf3 95
+closure-compiler 5e378edf1c53755fd7c97dcf917b4b7aa3187655 96
+closure-compiler c74e669d2a83734ea967d8b65dab2b82e9278aac 97
+closure-compiler 966054fce742e5b47b65ee4f78f8c30ecda3c293 98
+closure-compiler 2e24e457dc36784081173e49f6d845cd335ec668 99
+closure-compiler 6894977799fa8dcff750892bd39a9d1e022a81f1 100
+closure-compiler 539245c382157319b305b8698dbd7e81534a482e 101
+closure-compiler f700c8f33214428944ef0d7d2fe21b2651565b3a 102
+closure-compiler 65bd660d8549d757f08c9b2bb38b2c88187bbf7e 103
+closure-compiler b12fbbe91a771ea3b8df79e2efe1a20c9f05b19d 104
+closure-compiler 888311d30b5b94398a8a81aa8ae6b528f3ee942b 105
+closure-compiler a98bcd9815c03603a6a6505f6796dfbdc2454808 106
+closure-compiler e3c0460f6e65e46a3a159595fa79c5a210755469 107
+closure-compiler ff5e3383cd4c52bd4e5d8eb9046078c44d0148f4 108
+closure-compiler 3de2b609ddd8289756678b1490d339440c442baa 109
+closure-compiler b2afc9296cff615bce51ab810fd854ab7ff002f8 110
+closure-compiler d541a0e1228619dbc59421ddff63e8e7a630e503 111
+closure-compiler 2ad03d8c7f78aca63cfc80c4a864c7517a7346e5 112
+closure-compiler 2cc09b1113e3ff8d55611663fe972e136c0bd9ca 113
+closure-compiler 3f48493d6f53792ea8ddcc1bccfccc758edfbd1e 114
+closure-compiler 500cdfa3d162b5aa23e520860e6a491c7ceab91c 115
+closure-compiler d5364b281207da57f60aa0c941de2602066d720e 116
+closure-compiler 5733617738edfa3f0554f7a548fe5a03b032c000 117
+closure-compiler 32adfc3649b169f300083f9493ec5dc43a997ecf 118
+closure-compiler 1d184abd606a447f83a7101b256c3df38c63a072 119
+closure-compiler d7160f7625652ca1f0379f80713855951d91d64e 120
+closure-compiler ffb967ffd4a822e7df0a63fb822670c63f1233ef 121
+closure-compiler cde410e9d8516324b9ddc973f89a4fcbed17893c 122
+closure-compiler 9c7eaf9392daaf7b4b10ae936f09a40e4a2aee63 123
+closure-compiler e976288b126f4d7dbb3e06428b97228c49522aa9 124
+closure-compiler 8b92a40c959420ec8fb4b5638f213d936c6c7c42 125
+closure-compiler 05b7fdce3269e6b843d0413aa8843f1863ebf38a 126
+closure-compiler 6e0bc15dd230ba24dec0b1991ad9dbe6dacd0f9a 127
+closure-compiler c160912293ed05794b3dc8ee1ff041e4d663d2f4 128
+closure-compiler 78aa4413506eb5db7ffccf2dff71876dd1f20705 129
+closure-compiler 7aa45b5fdca133ace5b44b335d754dad9b6a798c 130
+closure-compiler be1e4a9d70d98c28e31f7058c9d05a3612757c51 131
+closure-compiler 456cfa751b92110cf0df40c497f9c6b6ca36c332 132
+closure-compiler 22f095fc9f7b340e146472065cc787e268fb58be 133
+joda-time ec122e8aa33bede69f988caf74acb3726448e9d7 1
+joda-time 4a19d3608b2024928986cd65edf7ab0fe7c5a28b 2
+joda-time bdd6524946b6fb7d8809e05063de435679f3e03f 3
+joda-time 6aa2d59e32ac47e0977aa688e41530a8ca21cbe1 4
+joda-time a700f5d3ccdf92965a4a8e7a4a19ce611008c6b8 5
+joda-time 1c8442d09651d8f401ee9fc643f77d7cd4fdefa0 6
+joda-time 6e3846b41579f94f285e603e11e894dcbed3adf8 7
+joda-time a67a8efd447fe938db6f037e5166dc0e4ed22961 8
+joda-time 1881be3082303e3cfb45db0ee5f8c62515ca7dcb 9
+joda-time 3ccc42c6e4f52f4c45c4a89263afedc22cf121cc 10
+joda-time 4996354642357deeaeedf8c6198d23610ed64852 11
+joda-time cc6c0bbd72dd70de3a909895bc1f0cf054a0d6d9 12
+joda-time eeadddb5779d2e65e720cbd790b0ebf9985d32a4 13
+joda-time 1b2758798b2e1b9a685f3bd4c8ad1f30e136d7fb 14
+joda-time 6a45f7de8f4b7fde91483ef27bd805092a0121be 15
+joda-time 93812d0ea762746d7d48d7f0dc3a627736527475 16
+joda-time d3ed279e593bf3a61b284329f05d2a48c121958b 17
+joda-time 8edc4c9c472ea7fface45851e4034e33dd5ba3ff 18
+joda-time 4fe01f3c1ad7cf47519628d058016c57fdc15be5 19
+joda-time 3a491c148117ddc3f7ea94e3971be6a9b9262782 20
+joda-time 0cba1a38361004865eed5d47020daf14af8e0438 21
+joda-time 7f2c756944064e62b3657b6d906dab13f260d660 22
+joda-time 9dcfb85d12a59caef9b2a360faed1a235012252c 23
+joda-time 12d57cb11e126f7428194c936e35794a68af335d 24
+joda-time 88a7b22e7b98156c491f8e947d54b8c3e41d7f41 25
+joda-time 1a302a375f51718f8aab09e76c2cad5b37ff480d 26
+joda-time 46568689725bfa95a56376f5806321bf45304a6d 27
+mockito b3377f3869651f00a16201485eae3eb85b3d5719 1
+mockito 9984fe302e25596097f3d7230d46b8ca0d5ab893 2
+mockito 437ac4700086bf6222f8a326bacf1f04794ae26e 3
+mockito dc0875a17c7540f23b4f90ece9d9ca7112e18c8d 4
+mockito 224657141cd2dadca0828914a0f9ffed6c7b6a0c 5
+mockito 23ff83d05aeffb0d627310471ec3a0afb116bedf 6
+mockito 6215e45cb07f3bc4e485922fde13dfa011f8e0be 7
+mockito 60e43ebf708cf79a42eef7d4a77863c28042b96f 8
+mockito a87b9c033e923b00ec4e452d9c790a20359ff5e6 9
+mockito 5ce1efae8a7e51ecdf7a600e10e5335c600dea14 10
+mockito 5bc6a912f5ab518fa7f12e757578d9bff2f7948a 11
+mockito d11e7468214d5e30f5c7b2bc6c1b4866faeef0e5 12
+mockito b8dd672ed7cc497ad5c57faf8d78dfb7d33c49d3 13
+mockito d60170cd7f119e9169b0f78e5a8c4ab48567fa11 14
+mockito a1550f633021e792ae330bf8b6b7950655a74507 15
+mockito ee22441f8734b1aba47c2afd8f71cac52d3cc3b9 16
+mockito c8a1b678ecd619ed6eea24e0af915559447e77ab 17
+mockito a5a1fdbf8e50ac6e1e80c604a9d5ad314c1f0d0c 18
+mockito 63a68c6745fbfa74b885cc134ee4c0fcb5365eee 19
+mockito 99e5d9f2bccdad861a085d96e32ad83c756898ab 20
+mockito 4b3dc4c61a99f3ecc0ea86696b972c351b18aea7 21
+mockito ab0fa1f3ade5664d78ee1e62c89f58ce4f10d6ab 22
+mockito c0be82920713ef72a92476a389daf004b871fba6 23
+mockito 316d67e84ba278735d624cacb8e854a4dacdd8fd 24
+mockito cf013e5298b83f35cbb24e5a49817ba946591351 25
+mockito 09ba2e1e836f0b0e577bd86e66a926073542a714 26
+mockito f90807ee7e40a0cc14ae903d83bd88b89c4fa5f7 27
+mockito ee062e820fa8ad745d5fb277069c865b8e22b153 28
+mockito 54866fe6b956a8bbcfe07d14da19d6a8f1211fd5 29
+mockito 1e6d0be6e9994f2fa96e86c87a3d5e471c2e6ef5 30
+mockito 15c0846ebcc8e0b4e8767123d9c73e8f035678ed 31
+mockito 4be463baaf769e5677c52960c0d87aa600a05e16 32
+mockito 35dac70f60899891b36c6562e3c08b888689b03f 33
+mockito d380b8d4e0102adbe6202bc7e5c2d2cfc53d3d15 34
+mockito 00a8aac765d8656f543e6d559ed913d2f06154af 35
+mockito 5cf7c995434bdba5071599870ed2124b9adf9fae 36
+mockito 7f091fd73cdf4cd6d6507e093c2c9ce511cf1c3c 37
+mockito b33ecab187988b3e7505a101857c753a7acee312 38
+jfreechart df055bf81b7aa846530eadb7904d911cf84372c9 1
+jfreechart d88f6abee52d732fd9cc3d87608295e09ce3d7ef 2
+jfreechart 09ba9fb28b03a9b3d03574b5e8dcc6ec53b39a9c 3
+jfreechart 914e7c821279b50a891ea70a2c7e312e47c5be24 4
+jfreechart 21cd197829977fd9cdac859dd5dbbc0006b83692 5
+jfreechart da5b09094712c93928db37bd7c0f890ddb6473c1 6
+jfreechart f57524dd27d24fa8ef5390e7deb126e2e2d53d56 7
+jfreechart 6928865fe9e24db3216af491959f8f277f19ad16 8
+jfreechart f2945dc8b4dc4b3813fd964c762583bf03eca06b 9
+jfreechart 0a4ca0aaac4c6213ba9e8a7b3840cd3a34200747 10
+jfreechart f5b58bbf7b96f8dab33cce1ee73d81db027e4ae5 11
+jfreechart b0477c6d0a40c3b4b2488c5a3d883deb6ec9c582 12
+jfreechart 6672934f970278f93afc251a511b69497d8f8db0 13
+jfreechart 02bc4cf4ef16da5966053e33fb78f932ad4c3479 14
+jfreechart 167ffb6c5bc21f024c9333b0ba534148417426c7 15
+jfreechart bd272819bc271bf8a8a9992f2ad92250cf76dbf0 16
+jfreechart 50329cfaee901921fe675418a4f693e9f8f00460 17
+jfreechart 6245571f70d1e3354c06e2b155818a8e7008b34e 18
+jfreechart 79b5bc884494cd22d9dde31ccdb159d74fc35a87 19
+jfreechart fe4335a71434b8d47e0020b55dbf390c7e91e3d6 20
+jfreechart fc73be32a4853eb9ba7b38d5e4860c5905a3576b 21
+jfreechart 353a111fb53f78f2e3d4164123be357f1316940f 22
+jfreechart 1c0dc741561d7425d57d7b247557c0ccbb3f3db3 23
+jfreechart 1e79f8b4ddc49437d1ea51633b80e91952c1863f 24
+jfreechart d190ef064ad2b5c0947a6d960c1f3116039dc681 25
+jfreechart 41cbcc677a96f2a11521401db27f5d993ab8fc0e 26

python/mp0360.uni.lux.config.yml

@@ -0,0 +1,20 @@
+java:
+    8home:  /Library/Java/JavaVirtualMachines/jdk1.8.0_181.jdk/Contents/Home
+spinfer:
+    home:   /Users/anil.koyuncu/projects/fixminer/spinfer/spinfer.native
+    #home:   /Users/anil.koyuncu/projects/spinfer/spinfer.native
+dataset:
+#    home:   /Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/
+    home:   /Users/anil.koyuncu/projects/fixminer/fixminer-data/patches/
+#    home:   /data/fixminer-core/python/data/gumInputLinux/
+    repo:   /Users/anil.koyuncu/projects/datasets
+
+coccinelle:
+    home:   /Users/anil.koyuncu/projects/fixminer/spinfer/statics
+    #home:   /Users/anil.koyuncu/projects/spinfer/statics
+
+
+fixminer:
+#    datapath: /Users/anil.koyuncu/projects/test/fixminer-core/python/data/
+    datapath: /Users/anil.koyuncu/projects/fixminer/fixminer-data
+#    datapath: /data/fixminer-core/python/data/

python/otherDatasets.py

@@ -0,0 +1,313 @@
+from common.commons import *
+DATA_PATH = os.environ["DATA_PATH"]
+COMMIT_DFS = os.environ["COMMIT_DFS"]
+# DATASET_PATH = '/Users/anilkoyuncu/projects/datasets'
+DATASET_PATH = os.environ["REPO_PATH"]
+DATASET = os.environ["dataset"]
+ROOT = os.environ["ROOT_DIR"]
+
+def filetype_fileter(filename):
+    # return filename.endswith(u'.java') and not bool(re.search('test.*\/', filename))
+    return filename.endswith(u'.c') or filename.endswith(u'.h')
+
+
+
+def checkoutFiles(sha,shaOld, filePath,type, repo=None):
+    try:
+        # folderDiff = join(DATA_PATH, 'gumInput',repoName, 'DiffEntries')
+        folderDiff = join(type, 'DiffEntries')
+        folderPrev = join(type, 'prevFiles')
+        folderRev = join( type, 'revFiles')
+        if not os.path.exists(folderDiff):
+            os.mkdir(folderDiff)
+
+        if not os.path.exists(folderPrev):
+            os.mkdir(folderPrev)
+
+        if not os.path.exists(folderRev):
+            os.mkdir(folderRev)
+
+        # if repo is None:
+        #     repo = join(REPO_PATH,repoName)
+
+
+        savePath = filePath.replace('/','#')
+
+        if not isfile(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath + '.txt'):
+
+            cmd = 'git -C ' + repo + ' diff -U ' + shaOld + ':' + filePath + '..' + sha + ':' + filePath  # + '> ' + folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt')
+
+            output,errors = shellGitCheckout(cmd,enc='latin1')
+            if errors:
+                # print(errors)
+                raise FileNotFoundError
+
+            regex = r"@@\s\-\d+,*\d*\s\+\d+,*\d*\s@@ ?(.*\n)*"
+            match = re.search(regex, output)
+            if not match:
+                return
+                # print()
+            not_matched, matched = output[:match.start()], match.group()
+            numberOfHunks = re.findall('@@\s\-\d+,*\d*\s\+\d+,*\d*\s@@', matched)
+            if len(numberOfHunks) == 0:
+                return
+            diffFile = shaOld + '\n' + matched.replace(' @@ ', ' @@\n')
+
+            with open(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath + '.txt',
+                      'w') as writeFile:
+                writeFile.writelines(diffFile)
+
+
+
+            cmd = 'git -C ' + repo + ' show ' + sha + ':' + filePath + '> ' + folderRev + '/' + sha + '_' + shaOld + '_' +savePath
+
+            if errors:
+                # print(errors)
+                raise FileNotFoundError
+            o,errors= shellGitCheckout(cmd,enc='latin1')
+            cmd = 'git -C ' + repo + ' show ' + shaOld + ':' + filePath + '> ' + folderPrev + '/' + 'prev_'+sha + '_' + shaOld + '_' +savePath
+            if errors:
+                # print(errors)
+                raise FileNotFoundError
+
+            o,errors = shellGitCheckout(cmd,enc='latin1')
+            if errors:
+                # print(errors)
+                raise FileNotFoundError
+
+    except FileNotFoundError as fnfe:
+        if isfile(folderRev + '/' + sha + '_' + shaOld + '_' +savePath):
+            os.remove(folderRev + '/' + sha + '_' + shaOld + '_' +savePath)
+        if isfile(folderPrev + '/' + 'prev_'+sha + '_' + shaOld + '_' +savePath):
+            os.remove(folderPrev + '/' + 'prev_'+sha + '_' + shaOld + '_' +savePath)
+        if isfile(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt')):
+            os.remove(folderDiff + '/' + sha + '_' + shaOld + '_' + savePath.replace('.java','.txt'))
+        # print(fnfe)
+        # raise Exception(fnfe)
+    except Exception as e:
+        # print(e)
+        raise Exception(e)
+
+
+def prepareFiles(t,dsName):
+    try:
+        sha,files = t
+
+        shaOld = sha + '^'
+        # repo = '/Users/anil.koyuncu/projects/linux'
+        # repo = join(REPO_PATH,repoName)
+        gumInputRepo = join(DATASET,dsName)
+        if not os.path.exists(join(gumInputRepo)):
+            os.makedirs(gumInputRepo)
+
+        # cmd = 'git -C ' + repo + ' diff --name-only ' + shaOld + '..'+sha
+        #
+        # output, errors = shellGitCheckout(cmd, 'latin1')
+        # files = output.strip().split('\n')
+
+        # if len(nonJava) > 0:
+        #     logging.warning('Skipping commit %s',sha)
+        #     return
+        # if len(files) != 1:
+        #     return
+
+        # nonTest = [f for f in files if not re.search('test', f, re.I) and f.endswith('.c') or f.endswith(u'.h')]
+        #
+        # if len(nonTest) > 1:
+        #     return
+
+        nonTest = []
+        for k,v in files.items():
+            if v == 'M':
+                if k.endswith('.c') or k.endswith(u'.h'):
+                   nonTest.append(k)
+        # nonTest = [f for f in files.keys() if f.endswith('.c') or f.endswith(u'.h')]
+
+        cmd = 'git -C ' + join(DATASET_PATH,dsName) + ' rev-parse --short=6 ' + shaOld
+
+        output, errors = shellGitCheckout(cmd, enc='latin1')
+        shaOld = output.strip()
+
+        cmd = 'git -C ' + join(DATASET_PATH,dsName) + ' rev-parse --short=6 ' + sha
+        output, errors = shellGitCheckout(cmd, enc='latin1')
+        sha = output.strip()
+
+        if isinstance(nonTest, list):
+            for file in nonTest:
+                checkoutFiles(sha,shaOld,  file,gumInputRepo,join(DATASET_PATH,dsName))
+
+
+
+    except Exception as e:
+        print(e)
+
+
+def checkCommitLog(x,dsName):
+    # repo = '/Users/anil.koyuncu/projects/linux'
+    cmd= 'git -C ' + join(DATASET_PATH,dsName) + ' show ' + x + " --pretty=\"format:\" --name-status -M100%"
+
+    out, err = shellGitCheckout(cmd, enc='latin1')
+    log = {}
+    lines = out.strip().split('\n')
+    for line in lines:
+        fname = line[2:].strip()
+        ftype = line[:1]
+        log[fname] = ftype
+    log
+    df = pd.DataFrame(data=[[log,  x]], columns=['files', 'commit'])
+    return df
+
+def getCommitLog(x,dsName):
+    # repo = '/Users/anil.koyuncu/projects/linux'
+    # commit, repo = x
+
+    cmd = 'git -C ' + join(DATASET_PATH,dsName) + '/ ' + " show --pretty=format:'%B' --no-patch " + x
+
+    output = shellCallTemplate(cmd, 'latin-1')
+
+    # matches = re.finditer(r"\bfix[a-zA-Z]*", output,re.I)
+    matches = re.finditer(r"\bfix[a-zA-Z]*|\bbug[a-zA-Z]*", output,re.I)
+    match = list(matches)
+    fixes = []
+    if len(match) >= 1:
+        for m in match:
+            fixes.append(m.group())
+    #     return match[0].group()
+
+    # matches = re.finditer('http[s]?:\/\/.*show_bug\.cgi\?id=[0-9]*',output)
+    # match = list(matches)
+    # links = []
+    # if len(match) >= 1:
+    #     for m in match:
+    #         links.append(m.group())
+
+    df = pd.DataFrame(data=[[fixes, output,x]], columns=['fixes','log','commit'])
+    # df = df.T
+    # df.columns = ['log', 'commit']
+
+    return df
+
+
+
+    output
+
+def collectBugFixPatches(dsName):
+    commits = getAllCommits(dsName)
+    # remove commits that are only deleting or adding files
+    commits = commits[commits.files.apply(lambda x: np.any([i == 'M' for i in x.values()]))]
+    # keep only commits that are changing c files (.c)
+    commits = commits[commits.files.apply(lambda x: np.all([i.endswith('.c') for i in x.keys()]))]
+    #not a revert commit
+    # commits = commits[~commits.log.apply(lambda x: x.startswith('Revert'))]
+    # commits = commits[commits.files.apply(lambda x: len(x) == 1)]
+    # commits['cocci'] = commits.log.apply(lambda x: True if re.search('cocci|coccinelle', x) else False)
+    # coccis = commits[commits.cocci].commit.values.tolist()
+    fixes = commits[commits.fixes.str.len()!=0].commit.values.tolist()
+    # links = commits[commits.links.str.len()!=0].commit.values.tolist()
+
+    # bugs = set(fixes).union(links).union(coccis)
+    # bugs = set(fixes)#.union(coccis)
+    commits = commits[commits.commit.isin(fixes)]
+    print(len(commits))
+    # for s in a.commit.values.tolist():
+
+    parallelRun(prepareFiles,commits[['commit','files']].values.tolist(),dsName)
+        # prepareFiles(s)
+
+
+def markBugFixingPatches(commits,dsName):
+    # from pandarallel import pandarallel
+    #
+    # pandarallel.initialize()
+    # commits['files'] = commits.commit.parallel_apply(checkCommitLog)
+    # commits
+
+    f = parallelRunMergeNew(checkCommitLog, commits['commit'].values.tolist(), dsName)
+    res = pd.merge(commits, f, on=['commit'])
+    commits=res
+    #
+    # # commits['isC'] = commits.files.apply(lambda x:np.any([i.endswith('.c') or i.endswith('.h') for i in x.keys() ]))
+    # commits['isC'] = commits.files.apply(lambda x:np.all([i.endswith('.c') for i in x.keys() ]))
+    #
+    # commits = commits[commits.isC == True]
+
+    # commits.commit.parallel_apply(getCommitLog)
+    f = parallelRunMergeNew(getCommitLog, commits['commit'].values.tolist(),dsName)
+
+    res = pd.merge(commits, f, on=['commit'])
+
+    save_zipped_pickle(res, join(COMMIT_DFS, dsName+'Fix' + ".pickle"))
+    return res
+
+
+def getAllCommits(datasetName):
+    if isfile(join(COMMIT_DFS,datasetName+'Fix.pickle')):
+        return load_zipped_pickle(join(COMMIT_DFS,datasetName+'Fix.pickle'))
+    else:
+
+        if isfile(join(COMMIT_DFS,datasetName+'.pickle')):
+            commits =  load_zipped_pickle(join(COMMIT_DFS,datasetName+'.pickle'))
+        else:
+            if not os.path.exists(COMMIT_DFS):
+                os.mkdir(COMMIT_DFS)
+
+
+            cmd = 'git -C ' + join(DATASET_PATH,datasetName) + " log --no-merges --pretty=format:'{\"commit\":\"%H\",\"commitDate\":\"%ci\",\"title\":\"%f\",\"committer\":\"%ce\"}' > " + join(COMMIT_DFS,datasetName + '.commits')
+            output = shellCallTemplate(cmd, enc='latin1')
+
+            from commitCollector import makeDF
+            rDF = makeDF(join(COMMIT_DFS,datasetName + '.commits'))
+            save_zipped_pickle(rDF, join(COMMIT_DFS, datasetName + ".pickle"))
+            # return rDF
+            commits = rDF
+        return markBugFixingPatches(commits,datasetName)
+
+
+def core():
+    datasets = pd.read_csv(join(ROOT,'data', 'datasets.csv'))
+    # repoList = ['FFmpeg','curl','nginx','openssl','redis','tmux','vlc']
+    pjs = listdir(DATASET_PATH)
+    # newRepo = ['php-src','libtiff','cpython']
+    pjs = [i for i in pjs if not (i.startswith('.') or i.startswith('codeflaws'))]
+    # pjs = [i for i in pjs if (i in newRepo )]
+    repoList = pjs
+    for repo,src in datasets.values.tolist():
+        cmd = 'git -C ' + DATASET_PATH + ' clone ' + src
+        shellCallTemplate(cmd)
+        logging.info(repo)
+        collectBugFixPatches(repo)
+
+def codeflaws():
+    cf = listdir(join(DATASET_PATH,'codeflaws'))
+
+    type = join(DATASET,'codeflaws')
+    folderDiff = join(type, 'DiffEntries')
+    folderPrev = join(type, 'prevFiles')
+    folderRev = join(type, 'revFiles')
+    if not os.path.exists(folderDiff):
+        os.makedirs(folderDiff)
+
+    if not os.path.exists(folderPrev):
+        os.makedirs(folderPrev)
+
+    if not os.path.exists(folderRev):
+        os.makedirs(folderRev)
+    cfBugs = [i for i in cf if os.path.isdir(join(DATASET_PATH,'codeflaws',i))]
+    for cfBug in cfBugs:
+        bugs = [i for i in listdir(join(DATASET_PATH,'codeflaws',cfBug)) if i.endswith('.c')]
+        bugs.sort()
+        if len(bugs) == 2:
+            s1 = bugs[0].replace('.c', '').split('-')
+            s2 = bugs[1].replace('.c', '').split('-')
+            prev = s1[-1]
+            rev = s2[-1]
+            bugName = '-'.join(s1[: -1])
+            shutil.copy(join(DATASET_PATH,'codeflaws',cfBug,bugs[0]),join(folderPrev,"prev_"+bugName+"-"+prev+"-"+rev+'.c'))
+            shutil.copy(join(DATASET_PATH,'codeflaws',cfBug,bugs[1]),join(folderRev,bugName+"-"+prev+"-"+rev+'.c'))
+            cmd = 'diff -u ' + join(DATASET_PATH,'codeflaws',cfBug,bugs[0]) + ' ' + join(DATASET_PATH,'codeflaws',cfBug,bugs[1])+ ' > ' + join(folderDiff,bugName+"-"+prev+"-"+rev+'.c.txt')
+            logging.info(cmd)
+            output, e = shellGitCheckout(cmd)
+            logging.info(output)
+        else:
+            print()
+

python/pairs.py

@@ -0,0 +1,363 @@
+from common.commons import *
+DATA_PATH = os.environ["DATA_PATH"]
+ROOT = os.environ["ROOT_DIR"]
+INNER_DATA_PATH = join(ROOT,'data')
+PROJECT_TYPE = os.environ["PROJECT_TYPE"]
+def importToken():
+    # global dbDir, portInner, redis_db, pairs, cluster, action, e, idx, v, key
+    dbDir = join(INNER_DATA_PATH, 'redis')
+    portInner = '6380'
+    dbDir = join(INNER_DATA_PATH, 'redis')
+    startDB(dbDir, portInner, "clusterl2-gumInputALL.rdb")
+    import redis
+    pairsToken = join(DATA_PATH, 'pairsToken')
+    redis_db = redis.StrictRedis(host="localhost", port=portInner, db=1)
+    pairs = get_filepaths(pairsToken, '.txt')
+    for pair in pairs:
+        split = pair.split("/")
+
+        shapeName = split[-4]
+        shapeSize = split[-3]
+        cluster = split[-2]
+        action = split[-1].replace('.txt', '')
+
+        # cmd ="bash " + join(DATA_PATH,'redisSingleImport.sh') + " " +  pair + " 6380 " +  shapeName + "-"+sizeCluster+"-"+actionCluster ;#+, portInner,f.getName()+"-"+pair.getName().split("\\.")[0]);
+        cmd = "bash " + join(INNER_DATA_PATH,
+                             'redisSingleImport.sh') + " " + pair + " 6380 " + shapeName + "-" + shapeSize + "-" + cluster + "-" + action;  # +, portInner,f.getName()+"-"+pair.getName().split("\\.")[0]);
+
+        o, e = shellGitCheckout(cmd)
+        o
+        indexFile = pair.replace('.txt', '.index')
+        with open(indexFile, 'r') as iFile:
+            idx = iFile.readlines()
+        for i in idx:
+            k, v = i.split(',')
+            key = shapeName + "-" + shapeSize + "-" + cluster + "-" + action + "-" + k
+            # redis_db.set(key, v.strip())
+            redis_db.hset('filenames', key, v.strip())
+
+def importAction(rootType):
+    # global dbDir, portInner, redis_db, pairs, pair, split, shapeName, shapeSize, cluster, cmd, o, e, indexFile, iFile, idx, i, k, v, key
+    dbDir = join(INNER_DATA_PATH, 'redis')
+    # portInner = '6380'
+    # startDB(dbDir, portInner, "clusterl1-gumInputALL.rdb")
+    portInner = '6399'
+    startDB(dbDir, portInner, PROJECT_TYPE)
+
+    import redis
+    # import pairs
+    pairsAction = join(DATA_PATH, 'pairsAction',rootType)
+    redis_db = redis.StrictRedis(host="localhost", port=portInner, db=1)
+    pairs = get_filepaths(pairsAction, '.txt')
+    for pair in pairs:
+        split = pair.split("/")
+        shapeName = split[-3]
+        shapeSize = split[-2]
+        cluster = split[-1].replace('.txt', '')
+        cmd = "bash " + join(INNER_DATA_PATH,
+                             'redisSingleImport.sh') + " " + pair + " 6399 " + shapeName + "-" + shapeSize + "-" + cluster;  # +, portInner,f.getName()+"-"+pair.getName().split("\\.")[0]);
+
+        o, e = shellGitCheckout(cmd)
+        print(o)
+        indexFile = pair.replace('.txt', '.index')
+        with open(indexFile, 'r') as iFile:
+            idx = iFile.readlines()
+        for i in idx:
+            k, v = i.split(',')
+            key = shapeName + "-" + shapeSize + "-" + cluster + "-" + k
+            # redis_db.set(key, v.strip())
+            redis_db.hset('filenames', key, v.strip())
+
+
+# def importShape():
+#     # global dbDir, portInner, redis_db, pairs, pair, split, shapeName, cmd, o, e, indexFile, iFile, idx, i, k, v, key
+#     dbDir = join(DATA_PATH, 'redis')
+#     portInner = '6380'
+#     startDB(dbDir, portInner, "clusterl1-gumInputALL.rdb")
+#     import redis
+#     pairsShapes = join(DATA_PATH, 'pairs')
+#     redis_db = redis.StrictRedis(host="localhost", port=portInner, db=1)
+#     pairs = get_filepaths(pairsShapes, '.index')
+#     l = []
+#     for pair in pairs:
+#         # split = pair.split("/")
+#         # shapeName = split[-2]
+#         # sizeCluster = split[-1].replace('.txt', '')
+#         # cmd = "bash " + join(DATA_PATH, 'redisSingleImport.sh') + " " + pair + " 6380 " + shapeName + "-" + sizeCluster;
+#         #
+#         # o, e = shellGitCheckout(cmd)
+#         # print(o)
+#         # indexFile = pair.replace('.txt', '.index')
+#         with open(pair, 'r') as iFile:
+#             idx = iFile.readlines()
+#             idx = [i.split(',')[1] for i in idx]
+#             l.append(idx)
+#     l = list(itertools.chain.from_iterable(l))
+#     l = [i for i in l if not (i.startswith('commons-math') or i.startswith('commons-lang') or i.startswith(
+#         'closure-compiler') or i.startswith('joda-time') or i.startswith('mockito') or i.startswith('jfreechart'))]
+#     l
+#
+#         # for i in idx:
+#         #     k, v = i.split(',')
+#         #     key = shapeName + "-" + sizeCluster + "-" + k
+#         #     redis_db.set(key, v.strip())
+
+
+def importShape():
+    # global dbDir, portInner, redis_db, pairs, pair, split, shapeName, cmd, o, e, indexFile, iFile, idx, i, k, v, key
+    dbDir = join(INNER_DATA_PATH, 'redis')
+    # portInner = '6380'
+    # startDB(dbDir, portInner, "clusterl0-gumInputALL.rdb")
+
+    portInner = '6399'
+    startDB(dbDir, portInner, PROJECT_TYPE)
+
+    import redis
+    pairsShapes = join(DATA_PATH, 'pairs')
+    redis_db = redis.StrictRedis(host="localhost", port=portInner, db=1)
+    pairs = get_filepaths(pairsShapes, '.txt')
+    for pair in pairs:
+        split = pair.split("/")
+        shapeName = split[-2]
+        sizeCluster = split[-1].replace('.txt', '')
+        cmd = "bash " + join(INNER_DATA_PATH, 'redisSingleImport.sh') + " " + pair + " 6399 " + shapeName + "-" + sizeCluster;
+
+        o, e = shellGitCheckout(cmd)
+        print(o)
+        indexFile = pair.replace('.txt', '.index')
+        with open(indexFile, 'r') as iFile:
+            idx = iFile.readlines()
+        for i in idx:
+            k, v = i.split(',')
+            key = shapeName + "-" + sizeCluster + "-" + k
+            #redis_db.set(key, v.strip())
+            redis_db.hset('filenames',key,v.strip())
+
+def tokenPairs():
+    global shapes, shape, sizes, clusters, cluster, actions, action, idx, val, pairs
+    shapes = listdir(join(DATA_PATH, 'actions'))
+    shapes = [f for f in shapes if isdir(join(DATA_PATH, 'actions', f))]
+    for shape in shapes:
+        sizes = listdir(join(DATA_PATH, 'actions', shape))
+        sizes = [f for f in sizes if isdir(join(DATA_PATH, 'actions', shape, f))]
+        for sf in sizes:
+            if sf.startswith('.'):
+                continue
+            if sf == '1':
+                continue
+            clusters = listdir(join(DATA_PATH, 'actions', shape, sf))
+            for cluster in clusters:
+                if cluster.startswith('.'):
+                    continue
+                actions = listdir(join(DATA_PATH, 'actions', shape, sf, cluster))
+                for action in actions:
+
+                    files = listdir(join(DATA_PATH, 'actions', shape, sf, cluster, action))
+                    indexCompared = []
+                    if not os.path.exists(join(DATA_PATH, 'pairsToken', shape, sf, cluster)):
+                        os.makedirs(join(DATA_PATH, 'pairsToken', shape, sf, cluster))
+
+                    infexFile = join(DATA_PATH, 'pairsToken', shape, sf, cluster, action + '.index')
+                    if isfile(infexFile):
+                        test = pd.read_csv(infexFile, header=None, index_col=0)
+                        test.rename(columns={1: 'filename'}, inplace=True)
+
+                        newFiles = [i for i in files if i not in test.filename.values.tolist()]
+                        for newFile in newFiles:
+                            test = test.append(pd.DataFrame(columns=['filename'], data=[newFile]), ignore_index=True)
+                        indexCompared = test.index.values.tolist()
+                        test.to_csv(infexFile, header=None)
+                    else:
+                        with open(infexFile, 'w') as out:
+                            # csv_out = csv.writer(out)
+
+                            for idx, val in enumerate(files):
+                                out.write(str(idx) + ',' + val + '\n')
+                                indexCompared.append(str(idx))
+
+                    pairs = list(itertools.combinations(indexCompared, 2))
+                    pairsFile = join(DATA_PATH, 'pairsToken', shape, sf, cluster, action + '.txt')
+                    if isfile(pairsFile):
+                        test = pd.read_csv(pairsFile, header=None)
+                        test['pairs'] = test.apply(lambda x: tuple([x[0], x[1]]), axis=1)
+                        newPairs = [i for i in pairs if i not in test['pairs'].values.tolist()]
+                        with open(pairsFile, 'w') as out:
+                            # csv_out = csv.writer(out)
+                            for row in newPairs:
+                                a, b = row
+                                out.write(a + ',' + b + '\n')
+                    else:
+                        with open(pairsFile, 'w') as out:
+                            # csv_out = csv.writer(out)
+                            for row in pairs:
+                                a, b = row
+                                out.write(a + ',' + b + '\n')
+
+def actionPairs(rootType):
+    # global shapes, shape, sizes, sf, clusters, cluster, files, indexCompared, out, idx, val, pairs, row, a, b
+    # shapes = listdir(join(DATA_PATH, 'shapes'))
+    # shapes = [f for f in shapes if isdir(join(DATA_PATH, 'shapes', f))]
+    shapes = [rootType]
+    for shape in shapes:
+        sizes = listdir(join(DATA_PATH, 'shapes', shape))
+        sizes = [f for f in sizes if isdir(join(DATA_PATH, 'shapes', shape, f))]
+        for sf in sizes:
+            if sf.startswith('.'):
+                continue
+            clusters = listdir(join(DATA_PATH, 'shapes', shape, sf))
+            for cluster in clusters:
+                if cluster.startswith('.'):
+                    continue
+                files = listdir(join(DATA_PATH, 'shapes', shape, sf, cluster))
+                indexCompared = []
+                if not os.path.exists(join(DATA_PATH, 'pairsAction', shape, sf)):
+                    os.makedirs(join(DATA_PATH, 'pairsAction', shape, sf))
+
+                indexFile = join(DATA_PATH, 'pairsAction', shape, sf, cluster + '.index')
+                if isfile(indexFile):
+                    test = pd.read_csv(indexFile, header=None, index_col=0)
+                    test.rename(columns={1: 'filename'}, inplace=True)
+
+                    newFiles = [i for i in files if i not in test.filename.values.tolist()]
+                    for newFile in newFiles:
+                        test = test.append(pd.DataFrame(columns=['filename'], data=[newFile]), ignore_index=True)
+                    indexCompared = test.index.values.tolist()
+                    test.to_csv(indexFile, header=None)
+                else:
+                    with open(indexFile, 'w') as out:
+                        # csv_out = csv.writer(out)
+
+                        for idx, val in enumerate(files):
+                            out.write(str(idx) + ',' + val + '\n')
+                            indexCompared.append(str(idx))
+
+                pairs = list(itertools.combinations(indexCompared, 2))
+
+                pairsFile = join(DATA_PATH, 'pairsAction', shape, sf, cluster + '.txt')
+                if isfile(pairsFile):
+                    test = pd.read_csv(pairsFile, header=None)
+                    test['pairs'] = test.apply(lambda x: tuple([x[0], x[1]]), axis=1)
+                    newPairs = [i for i in pairs if i not in test['pairs'].values.tolist()]
+                    with open(pairsFile, 'w') as out:
+                        # csv_out = csv.writer(out)
+                        for row in newPairs:
+                            a, b = row
+                            out.write(a + ',' + b + '\n')
+                else:
+                    with open(pairsFile, 'w') as out:
+                        # csv_out = csv.writer(out)
+                        for row in pairs:
+                            a, b = row
+                            out.write(a + ',' + b + '\n')
+
+def shapePairs():
+    # global dbDir, portInner, redis_db, keys, matches, roots, sizes, sf, files, indexCompared, out, idx, val, pairs, row, a, b
+    # if not (isfile(join(DATA_PATH, 'studyDataset.pickle'))):
+    dbDir = join(INNER_DATA_PATH, 'redis')
+
+    portInner = '6399'
+    startDB(dbDir, portInner, PROJECT_TYPE)
+
+    import redis
+    redis_db = redis.StrictRedis(host="localhost", port=portInner, db=0)
+    keys = redis_db.hkeys("dump")#hkeys "dump"
+    # keys = redis_db.scan(0, match='*', count='1000000')
+
+    matches = pd.DataFrame(keys, columns=['pairs_key'])
+
+    # matches = load_zipped_pickle(join(DATA_PATH,'singleHunks'))
+    matches['pairs_key'] = matches['pairs_key'].apply(lambda x: x.decode())
+    matches['root'] = matches['pairs_key'].apply(lambda x: x.split('/')[0])
+    matches['size'] = matches['pairs_key'].apply(lambda x: x.split('/')[1])
+    matches['file'] = matches['pairs_key'].apply(lambda x: x.split('/')[2])
+
+    # matches[matches.file.apply(
+    #     lambda i: (i.startswith('commons-math.git') or i.startswith('commons-lang.git') or i.startswith(
+    #         'closure-compiler.git') or i.startswith('joda-time.git') or i.startswith('mockito.git')))]
+    matches['fileName'] = matches['pairs_key'].apply(lambda x: '_'.join(x.split('/')[2].split('_')[:-1]))
+    # else:
+
+        # matches = load_zipped_pickle(join(DATA_PATH, 'studyDataset.pickle'))
+        # matches = matches[matches.repo.apply(lambda i: not (
+        #             i.startswith('commons-math') or i.startswith('commons-lang') or i.startswith(
+        #         'closure-compiler.git') or i.startswith('joda-time.git') or i.startswith('mockito.git')))]
+
+    # matches = matches[matches['size'] != '1']
+    matches['hunk'] = matches['pairs_key'].apply(lambda x: x.split('/')[2].split('_')[-1])
+    # test = matches[['fileName', 'hunk']]
+    # df = test.groupby(by=['fileName'], as_index=False).agg(lambda x: x.tolist())
+    # sDF = df[df.hunk.apply(lambda x: True if x == ['0'] else False)]
+    # # sDF = df[df.hunk.apply(lambda x: True if len(x)<10005 else False)]
+    # singleHunkedFiles = sDF.fileName.unique().tolist()
+    # # singleHunkedFiles = [i.replace('.txt', '') for i in singleHunkedFiles]
+    # matches = matches[matches.fileName.isin(singleHunkedFiles)]
+
+
+    return matches
+
+def createPairs(matches):
+
+    if len(matches) == 0:
+        return True
+    roots = matches.root.unique().tolist()
+    # roots = listdir(join(DATA_PATH,'EnhancedASTDiffgumInput'))
+    if os.path.exists(join(DATA_PATH, 'pairs')):
+        import shutil
+        shutil.rmtree(join(DATA_PATH, 'pairs'))
+    for root in roots:
+        # if root.startswith('.'):
+        #     continue
+        rootMatch = matches[matches['root'] == root]
+        sizes = rootMatch['size'].unique().tolist()
+        for sf in sizes:
+            # if sf == '1':
+            #     continue
+            match = rootMatch[rootMatch['size'] == sf]
+            files = match.file.unique().tolist()
+
+            if len(files) > 1:
+                files
+                indexCompared = []
+                if not os.path.exists(join(DATA_PATH, 'pairs', root)):
+                    os.makedirs(join(DATA_PATH, 'pairs', root))
+
+
+                indexFile = join(DATA_PATH, 'pairs', root, sf + '.index')
+                if isfile(indexFile):
+                    test = pd.read_csv(indexFile, header=None, index_col=0)
+                    test.rename(columns={1: 'filename'}, inplace=True)
+
+                    newFiles = [i for i in files if i not in test.filename.values.tolist()]
+                    for newFile in newFiles:
+                        test = test.append(pd.DataFrame(columns=['filename'], data=[newFile]), ignore_index=True)
+                    indexCompared = test.index.values.tolist()
+                    test.to_csv(indexFile, header=None)
+                else:
+
+                    with open(indexFile, 'w') as out:
+                        # csv_out = csv.writer(out)
+
+                        for idx, val in enumerate(files):
+                            out.write(str(idx) + ',' + val + '\n')
+                            indexCompared.append(str(idx))
+
+                pairs = list(itertools.combinations(indexCompared, 2))
+
+                # import csv
+                pairsFile = join(DATA_PATH, 'pairs', root, sf + '.txt')
+                if isfile(pairsFile):
+                    test = pd.read_csv(pairsFile, header=None)
+                    test['pairs'] = test.apply(lambda x: tuple([x[0], x[1]]), axis=1)
+                    newPairs = [i for i in pairs if i not in test['pairs'].values.tolist()]
+                    with open(pairsFile, 'w') as out:
+                        # csv_out = csv.writer(out)
+                        for row in newPairs:
+                            a, b = row
+                            out.write(a + ',' + b + '\n')
+                else:
+                    with open(pairsFile, 'w') as out:
+                        # csv_out = csv.writer(out)
+                        for row in pairs:
+                            a, b = row
+                            out.write(a + ',' + b + '\n')
+    return False

python/patchManyBugs.py

@@ -0,0 +1,206 @@
+from common.commons import *
+DATA_PATH = os.environ["DATA_PATH"]
+SPINFER_PATH = os.environ["spinfer"]
+ROOT_DIR = os.environ["ROOT_DIR"]
+SPINFER_INDEX_PATH = os.environ["dataset"]
+COCCI_PATH = join(os.environ["coccinelle"],'spatch')
+DATASET = os.environ["dataset"]
+
+import bugzoo
+from bugzoo import server, Container
+from bugzoo import Patch, Client
+import signal
+
+sosbugs = ['manybugs:gmp:14166-14167', 'manybugs:gzip:2009-08-16-3fe0caeada-39a362ae9d',
+           'manybugs:gzip:2009-10-09-1a085b1446-118a107f2d', 'manybugs:gzip:2010-02-19-3eb6091d69-884ef6d16c',
+           'manybugs:libtiff:2005-12-21-3b848a7-3edb9cd', 'manybugs:libtiff:2006-03-03-a72cf60-0a36d7f',
+           'manybugs:libtiff:2006-03-03-eec4c06-ee65c74', 'manybugs:libtiff:2007-07-13-09e8220-f2d989d',
+           'manybugs:libtiff:2007-11-02-371336d-865f7b2', 'manybugs:libtiff:2009-02-05-764dbba-2e42d63',
+           'manybugs:libtiff:2009-08-28-e8a47d4-023b6df', 'manybugs:libtiff:2010-11-27-eb326f9-eec7ec0',
+           'manybugs:libtiff:2006-02-23-b2ce5d8-207c78a', 'manybugs:lighttpd:2661-2662', 'manybugs:lighttpd:2254-2259',
+           'manybugs:lighttpd:2785-2786', 'manybugs:lighttpd:1948-1949',
+           'manybugs:php:2011-12-10-74343ca506-52c36e60c4', 'manybugs:php:2011-04-02-70075bc84c-5a8c917c37',
+           'manybugs:php:2011-03-25-8138f7de40-3acdca4703', 'manybugs:php:2011-12-04-1e6a82a1cf-dfa08dc325',
+           'manybugs:php:2012-02-08-ff63c09e6f-6672171672', 'manybugs:php:2011-11-19-eeba0b5681-d3b20b4058',
+           'manybugs:php:2011-04-07-77ed819430-efcb9a71cd', 'manybugs:php:2011-02-01-01745fa657-1f49902999',
+           'manybugs:php:2012-03-12-7aefbf70a8-efc94f3115', 'manybugs:php:2011-10-15-0a1cc5f01c-05c5c8958e',
+           'manybugs:php:2011-01-30-5bb0a44e06-1e91069eb4', 'manybugs:php:2011-02-01-fefe9fc5c7-0927309852',
+           'manybugs:php:2011-02-27-e65d361fde-1d984a7ffd', 'manybugs:php:2011-03-19-5d0c948296-8deb11c0c3',
+           'manybugs:php:2011-03-23-63673a533f-2adf58cfcf', 'manybugs:php:2011-04-06-187eb235fe-2e25ec9eb7',
+           'manybugs:php:2011-04-09-db01e840c2-09b990f499', 'manybugs:php:2011-05-17-453c954f8a-daecb2c0f4',
+           'manybugs:php:2011-05-24-b60f6774dc-1056c57fa9', 'manybugs:php:2011-10-16-1f78177e2b-d4ae4e79db',
+           'manybugs:php:2011-10-31-2e5d5e5ac6-b5f15ef561', 'manybugs:php:2011-10-31-c4eb5f2387-2e5d5e5ac6',
+           'manybugs:php:2011-11-01-ceac9dc490-9b0d73af1d', 'manybugs:php:2011-11-11-fcbfbea8d2-c1e510aea8',
+           'manybugs:php:2011-11-15-236120d80e-fb37f3b20d', 'manybugs:php:2011-11-16-55acfdf7bd-3c7a573a2c',
+           'manybugs:php:2011-11-22-ecc6c335c5-b548293b99', 'manybugs:php:2011-11-23-eca88d3064-db0888dfc1',
+           'manybugs:php:2012-01-27-544e36dfff-acaf9c5227', 'manybugs:php:2012-01-30-9de5b6dc7c-4dc8b1ad11',
+           'manybugs:php:2012-02-25-c1322d2505-cfa9c90b20', 'manybugs:php:2012-03-04-60dfd64bf2-34fe62619d',
+           'manybugs:php:2012-03-08-0169020e49-cdc512afb3', 'manybugs:php:2012-03-11-3954743813-d4f05fbffc',
+           'manybugs:php:2012-03-12-438a30f1e7-7337a901b7', 'manybugs:python:69223-69224',
+           'manybugs:python:69368-69372', 'manybugs:python:70098-70101', 'manybugs:python:70056-70059',
+           'manybugs:wireshark:37112-37111', 'manybugs:wireshark:37122-37123', 'manybugs:gmp:13420-13421',
+           'manybugs:gzip:2009-09-26-a1d3d4019d-f17cbd13a1', 'manybugs:lighttpd:1913-1914',
+           'manybugs:php:2011-11-19-51a4ae6576-bc810a443d', 'manybugs:php:2011-03-11-d890ece3fc-6e74d95f34',
+           'manybugs:php:2011-11-19-eeba0b5681-f330c8ab4e', 'manybugs:php:2012-01-01-80dd931d40-7c3177e5ab']
+
+
+def cocciCore(t):
+    cmd, manybug, patchName, spfile,srcPath = t
+    # logging.info(cmd)
+    output, e = shellGitCheckout(cmd)
+    # logging.info(output)
+    patchSize = os.path.getsize(join(DATA_PATH,"manybugs",manybug,'patches',patchName+spfile+'.txt'))
+    if patchSize == 0 :
+        os.remove(join(DATA_PATH,"manybugs",manybug,'patches',patchName+spfile+'.txt'))
+    else:
+        cmd = 'patch -d '+srcPath.replace(patchName,'') + ' -i ' + join(DATA_PATH,"manybugs",manybug,'patches',patchName+spfile+'.txt') + ' -o ' + join(DATA_PATH,"manybugs",manybug,'patched',patchName+spfile+'.c')
+        output, e = shellGitCheckout(cmd)
+        output,e
+        print(output)
+
+
+def buildAll():
+    bugList = []
+    port = 6000
+    for i in sosbugs:
+        t = i,port
+        bugList.append(t)
+        if port == 6300:
+            port = 6000
+        port += 1
+    parallelRun(checkBuild, bugList, max_workers=4)
+
+
+def checkBuild(t):
+    bugName, port = t
+    # port = 3000
+    # cmd = 'bash {} {}'.format(join(DATA_PATH, 'startBugzoo.sh'), port)
+    cmd = 'bash {} {}'.format(join(ROOT_DIR,'data', 'buildBugzoo.sh'), bugName)
+
+    out, e = shellGitCheckout(cmd)
+
+    # container = None
+    # with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) as process:
+    #
+    #     # o,e = shellGitCheckout(cmd)
+    #     url = "http://127.0.0.1:{}".format(port)
+    #     timeout_connection = 3000
+    #     client = Client(url, timeout_connection=timeout_connection)
+    #
+    #     try:
+    #         bug = client.bugs[bugName]
+    #         if not client.bugs.is_installed(bug):
+    #             print(' building ' + bugName)
+    #             o = client.bugs.build(bug)
+    #
+    #             # client.bugs.download(bug)
+    #             # print("the image is not installed :'(")
+    #
+    #         # print("creating container...")
+    #         container = client.containers.provision(bug)
+    #
+    #     except Exception as e:
+    #         logging.error(e)
+    #         # continue
+    #     finally:
+    #         # ''
+    #         cmd = 'docker stop {}'.format(container.id)
+    #         out, e = shellGitCheckout(cmd)
+    #         client.shutdown()
+    #         os.killpg(process.pid, signal.SIGTERM)
+
+def patchCore():
+    try:
+
+        pathManyBug = join(DATA_PATH, "manybugs")
+        manybugs = listdir(pathManyBug)
+
+        manybugs = [i for i in manybugs if i in sosbugs]
+        spfiles = listdir(join(DATASET,'cocci'))
+
+        from sprinferIndex import filterPatterns
+        filteredPattern = filterPatterns()
+
+        spfiles = [i for i in spfiles if i in filteredPattern]
+        workList = []
+        for manybug in manybugs:
+            if manybug == '.DS_Store':
+                continue
+            # files = listdir(join(join(DATA_PATH,"manybugs",manybug,'diffs')))
+            if  os.path.exists(join(pathManyBug, manybug, 'patches')):
+                shutil.rmtree(join(pathManyBug, manybug, 'patches'))
+
+                os.mkdir(join(pathManyBug, manybug, 'patches'))
+            else:
+                os.mkdir(join(pathManyBug, manybug, 'patches'))
+
+            if  os.path.exists(join(pathManyBug, manybug, 'patched')):
+                shutil.rmtree(join(pathManyBug, manybug, 'patched'))
+
+                os.mkdir(join(pathManyBug, manybug, 'patched'))
+            else:
+                os.mkdir(join(pathManyBug, manybug, 'patched'))
+
+            bug, fix = manybug.split(':')[-1].split('-')[-2:]
+            files = get_filepaths(join(pathManyBug, manybug,'diffs'), bug)
+            sources = [i for i in files if not i.endswith('oracle.c')]
+
+
+            for src in sources:
+                # srcPath = src.replace('/diffs/','/src/')
+                srcPath = src
+                patchName = src.split('/')[-1]
+                for spfile in spfiles:
+                     cmd = COCCI_PATH + ' --sp-file ' + join(DATASET,'cocci',spfile) + ' ' + srcPath + ' --patch -o' + join(pathManyBug,manybug,'patches',patchName) + ' > ' + join(pathManyBug, manybug, 'patches', patchName + spfile + '.txt')
+                     # cmd = COCCI_PATH + ' --sp-file ' + join(DATASET,'cocci',spfile) + ' ' + srcPath + ' -o ' + join(DATA_PATH,"introclass",manybug,'patches',patchName+spfile+'.txt')
+                     t = cmd,manybug,patchName,spfile,srcPath
+                     workList.append(t)
+    except Exception as e:
+        logging.error(e)
+    parallelRun(cocciCore,workList)
+
+
+def exportSosPatches():
+    pathManyBug = join(DATA_PATH, "manybugs")
+    pathManyBugSos = join(DATA_PATH, "manybugs_sos")
+    manybugs = listdir(pathManyBug)
+    manybugs = [i for i in manybugs if i in sosbugs]
+
+    for manybug in manybugs:
+        # shutil.copytree(join(pathManyBug,manybug,'patched'),join(pathManyBugSos,manybug,'patched'))
+        shutil.copytree(join(pathManyBug,manybug,'patches'),join(pathManyBugSos,manybug,'patches'))
+
+def patched():
+    pathManyBug = join(DATA_PATH, "manybugs")
+    manybugs = listdir(pathManyBug)
+    manybugs = [i for i in manybugs if i in sosbugs]
+    # spfiles = listdir(join(DATASET,'cocci'))
+    workList = []
+    for manybug in manybugs:
+        if manybug == '.DS_Store':
+            continue
+        # files = listdir(join(join(DATA_PATH,"manybugs",manybug,'diffs')))
+        if  os.path.exists(join(DATA_PATH, "manybugs_patched", manybug, 'patches')):
+            shutil.rmtree(join(DATA_PATH, "manybugs_patched", manybug, 'patches'))
+
+            os.makedirs(join(DATA_PATH, "manybugs_patched", manybug, 'patches'))
+        else:
+            os.makedirs(join(DATA_PATH, "manybugs_patched", manybug, 'patches'))
+        files = get_filepaths(join(DATA_PATH, "manybugs", manybug), '.c')
+        sources = [i for i in files if not i.endswith('oracle.c')]
+
+        bug, fix = manybug.split(':')[-1].split('-')[-2:]
+        files = get_filepaths(join(pathManyBug, manybug, 'diffs'), bug)
+        sources = [i for i in files if not i.endswith('oracle.c')]
+
+        for src in sources:
+            # srcPath = src.replace('/diffs/','/src/')
+            srcPath = src
+            patchName = src.split('/')[-1]
+            spfiles = listdir(join(DATA_PATH,"manybugs",manybug,'patches'))
+            for spfile in spfiles:
+                 spfile =spfile.replace(patchName,'').replace('.txt','')
+                 cmd = COCCI_PATH + ' --sp-file ' + join(DATASET,'cocci',spfile) + ' ' + srcPath + ' -o ' + join(DATA_PATH,"manybugs_patched",manybug,'patches',patchName+spfile+'.c')
+                 t = cmd,manybug,patchName,spfile
+                 workList.append(t)
+    parallelRun(cocciCore,workList)

python/patch_validate.py

@@ -0,0 +1,509 @@
+import bugzoo
+from bugzoo import server, Container
+import csv
+import os
+from common.commons import *
+DATA_PATH = os.environ["DATA_PATH"]
+
+def readTestSuite(testPath):
+    regex = r"([p|n0-9]+)\)"
+    with open(testPath,mode='r') as testFile:
+        test_str = testFile.read()
+    matches = re.finditer(regex, test_str, re.MULTILINE)
+
+    testList = []
+    for matchNum, match in enumerate(matches, start=1):
+
+         for groupNum in range(0, len(match.groups())):
+            groupNum = groupNum + 1
+            testList.append(match.group(groupNum))
+    return testList
+def myvalidateCore(t):
+    bugName, port = t
+    container = None
+    with bugzoo.server.ephemeral(port=port, verbose=False, timeout_connection=30000) as client:
+        try:
+
+            bug = client.bugs[bugName]
+            if client.bugs.is_installed(bug):
+                pass
+            else:
+                client.bugs.build(bug)
+            fix = 'failure'
+            output = ''
+            output += 'bugName:' + bugName + ', '
+            container = client.containers.provision(bug)
+            # output += 'First_test:'
+
+            preId, postId = bugName.split(':')[-1].split('-')[-2:]
+            originalBugs = get_filepaths(join(DATA_PATH, 'manybugs', bugName, 'diffs'), preId)
+            # for ob in originalBugs:
+            #     filepath = ob.split('diffs')[-1]
+            #     cmd = 'cp /experiment/diffs' + filepath + ' src' + filepath.replace('-' + preId, '')
+            #     client.containers.exec(container=container, command=cmd, context='/experiment/')
+
+            validTests = readTestSuite(join(DATA_PATH, 'manybugs', bugName, 'test.sh'))
+            output += 'total ' +  str(len(bug.tests._tests)) + ' newtotal ' + str(len(validTests))
+            # client.containers.build(container)
+            # pre_failure_cases, pre_failure, total, pre_test_outcomes = test_all(bug, container, client,validTests)
+            # if pre_failure == 0:
+            #     logging.error(bugName + ' no failed test initially')
+            #     return ''
+            # output += '@fail:' + ','.join(pre_failure) + '@total:' + str(total) + ', ' + '@newtotal:' + str(len(validTests)) + ', '
+            #
+            # fixBugs = get_filepaths(join(DATA_PATH, 'manybugs', bugName, 'diffs'), postId)
+            # for ob in fixBugs:
+            #     filepath = ob.split('diffs')[-1]
+            #     cmd = 'cp /experiment/diffs' + filepath + ' src' + filepath.replace('-' + postId, '')
+            #     client.containers.exec(container=container, command=cmd, context='/experiment/')
+            # client.containers.build(container)
+            # diff_files = get_diff(bug.name, client, container)
+            # patch_result = patch_application(client, container, diff_files)
+            # if not patch_result:
+            #     print("@{}@".format(patch_result))
+            #     continue
+            # print("@{}@".format(patch_result),end=' ')
+            #
+            #
+            # print("Second_test:",end=' ')
+            # post_test_outcomes = {}
+            # post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client,validTests)
+            # output += ','.join(post_failure) + ' '
+            # if len(post_failure) == 0:
+            #     # times += 1
+            #     fix = 'success'
+            #     # print("fix {} by {}".format(bugName, patch_name))
+            #     output += 'fix {}  '.format(bugName)
+
+
+            # patch_names = os.listdir(join(DATA_PATH,'manybugs',bugName,'patched'))
+            # patch_names = [i for i in patch_names if not i.endswith('.DS_Store')]
+            # if len(originalBugs) != 1:
+            #     members = []
+            #
+            #     for member in set([i.split('-')[0] for i in patch_names]):
+            #         members.append([i for i in patch_names if i.startswith(member)])
+            #
+            #     patch_names= list(itertools.product(*members))
+            #     patch_names
+            # times = 0
+            # for patch_name in patch_names:
+            #     if isinstance(patch_name, tuple):
+            #         for i in patch_name:
+            #             copyPatch(bugName, client, container, originalBugs, i, preId)
+            #     else:
+            #         copyPatch(bugName, client, container, originalBugs, patch_name, preId)
+            #
+            #     patch_result = client.containers.build(container)
+            #     if not patch_result.successful:
+            #         output += '@False@F '
+            #         continue
+            #     output += '@True@'
+            #
+            #     post_test_outcomes = {}
+            #     post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client,validTests)
+            #
+            #     output += ','.join(post_failure) + ' '
+            #     if len(post_failure) == 0:
+            #         # times += 1
+            #         fix = 'success'
+            #         # print("fix {} by {}".format(bugName, patch_name))
+            #         output += 'fix {}  '.format(bugName)
+            #
+            # output += 'times:{}, '.format(times) + fix
+
+            print(output)
+            return output
+
+        except Exception as e:
+            logging.error(e)
+            # continue
+        finally:
+            # ''
+            cmd = 'docker stop {}'.format(container.id)
+            out, e = shellGitCheckout(cmd)
+            # docker stop $(docker ps -q)
+
+def validateCore(t):
+    bugName, port = t
+    container = None
+    with bugzoo.server.ephemeral(port=port, verbose=False, timeout_connection=30000) as client:
+        try:
+
+            bug = client.bugs[bugName]
+            if client.bugs.is_installed(bug):
+                pass
+            else:
+                client.bugs.build(bug)
+            fix = 'failure'
+            output = ''
+            output += 'bugName:' + bugName + ', '
+            container = client.containers.provision(bug)
+            output += 'First_test:'
+
+
+            preId, postId = bugName.split(':')[-1].split('-')[-2:]
+
+            failing = join(DATA_PATH, 'manybugs', bugName, 'failing.tests.txt')
+            passing = join(DATA_PATH, 'manybugs', bugName, 'passing.tests.txt')
+
+            if not (isfile(failing)):
+                return
+
+            cmd = 'docker cp ' +failing+ ' '+container.id +':/experiment/failing.tests.txt '
+            o,e = shellGitCheckout(cmd)
+            cmd = 'sudo chown $(whoami):$(whoami) "{}"'
+            cmd = cmd.format('failing.tests.txt')
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/')
+            cmd = 'docker cp ' +passing+ ' '+container.id +':/experiment/passing.tests.txt '
+            o,e = shellGitCheckout(cmd)
+            cmd = 'sudo chown $(whoami):$(whoami) "{}"'
+            cmd = cmd.format('passing.tests.txt')
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/')
+
+            originalBugs = get_filepaths(join(DATA_PATH, 'manybugs', bugName, 'diffs'), preId)
+            for ob in originalBugs:
+                filepath = ob.split('diffs')[-1]
+                cmd = 'cp diffs' + filepath + ' src' + filepath.replace('-' + preId, '')
+                client.containers.exec(container=container, command=cmd, context='/experiment/')
+
+            validTests = readTestSuite(join(DATA_PATH, 'manybugs', bugName, 'test.sh'))
+            client.containers.build(container)
+            pre_failure_cases, pre_failure, total, pre_test_outcomes = test_all(bug, container, client,validTests)
+            if pre_failure == 0:
+                logging.error(bugName + ' no failed test initially')
+                return ''
+            output += '@fail:' + ','.join(pre_failure) + '@total:' + str(total) + ', ' + '@newtotal:' + str(len(validTests)) + ', '
+
+            fixBugs = get_filepaths(join(DATA_PATH, 'manybugs', bugName, 'diffs'), postId)
+            for ob in fixBugs:
+                filepath = ob.split('diffs')[-1]
+                cmd = 'cp diffs' + filepath + ' src' + filepath.replace('-' + postId, '')
+                client.containers.exec(container=container, command=cmd, context='/experiment/')
+            client.containers.build(container)
+            # diff_files = get_diff(bug.name, client, container)
+            # patch_result = patch_application(client, container, diff_files)
+            # if not patch_result:
+            #     print("@{}@".format(patch_result))
+            #     continue
+            # print("@{}@".format(patch_result),end=' ')
+            #
+            #
+            # print("Second_test:",end=' ')
+            # post_test_outcomes = {}
+            post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client,validTests)
+            output += ','.join(post_failure) + ' '
+            if len(post_failure) == 0:
+                # times += 1
+                fix = 'success'
+                # print("fix {} by {}".format(bugName, patch_name))
+                output += 'fix {}  '.format(bugName)
+
+
+            # patch_names = os.listdir(join(DATA_PATH,'manybugs',bugName,'patched'))
+            # patch_names = [i for i in patch_names if not i.endswith('.DS_Store')]
+            # if len(originalBugs) != 1:
+            #     members = []
+            #
+            #     for member in set([i.split('-')[0] for i in patch_names]):
+            #         members.append([i for i in patch_names if i.startswith(member)])
+            #
+            #     patch_names= list(itertools.product(*members))
+            #     patch_names
+            # times = 0
+            # for patch_name in patch_names:
+            #     if isinstance(patch_name, tuple):
+            #         for i in patch_name:
+            #             copyPatch(bugName, client, container, originalBugs, i, preId)
+            #     else:
+            #         copyPatch(bugName, client, container, originalBugs, patch_name, preId)
+            #
+            #     patch_result = client.containers.build(container)
+            #     if not patch_result.successful:
+            #         output += '@False@F '
+            #         continue
+            #     output += '@True@'
+            #
+            #     post_test_outcomes = {}
+            #     post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client)
+            #
+            #     output += str(post_failure) + ' '
+            #     if post_failure == 0:
+            #         times += 1
+            #         fix = 'success'
+            #         output += 'fix {} by {} '.format(bugName, patch_name)
+            #
+            # output += 'times:{}, '.format(times) + fix
+
+            print(output)
+            return output
+
+        except Exception as e:
+            logging.error(e)
+            # continue
+        finally:
+            # ''
+            cmd = 'docker stop {}'.format(container.id)
+            out, e = shellGitCheckout(cmd)
+            # docker stop $(docker ps -q)
+
+
+def copyPatch(bugName, client, container, originalBugs, patch_name, preId):
+    originalBugName = patch_name.split('-' + preId)[0]
+    originalBugPaths = [i.split('diffs')[-1].split(originalBugName)[0] for i in originalBugs if
+                        i.split('/')[-1].startswith(originalBugName)]
+    if len(originalBugPaths) == 1:
+        originalBugPath = originalBugPaths[0]
+    else:
+        logging.error('too many paths')
+    cmd = 'docker cp ' + join(DATA_PATH, 'manybugs', bugName, 'patched',
+                              patch_name) + ' ' + container.id + ':/experiment/src' + originalBugPath + originalBugName
+    o, e = shellGitCheckout(cmd)
+    cmd = 'sudo chown $(whoami):$(whoami) "{}"'
+    cmd = cmd.format('src' + originalBugPath + originalBugName)
+    o = client.containers.exec(container=container, command=cmd, context='/experiment/')
+
+    cmd = 'rm -rf "{}"'
+    cmd = cmd.format('src' + originalBugPath + originalBugName.replace('.c','.o'))
+    o = client.containers.exec(container=container, command=cmd, context='/experiment/')
+
+
+def patch_validate():
+    logger = logging.getLogger()
+
+    for k,v in logger.manager.loggerDict.items():
+        if(k.startswith('bugzoo')):
+            if isinstance(v,logging.Logger):
+                v.setLevel(logging.ERROR)
+
+    buglist = listdir(join(DATA_PATH,'manybugs'))
+
+    # for i in range(0,len(bugList)):
+    bugList = []
+    port = 6000
+    for b in buglist:
+        if b== '.DS_Store':
+            continue
+        t = b, port
+        if (b.startswith('manybugs:php:')):
+            bugList.append(t)
+            if port == 6300:
+                port = 6000
+            port += 1
+    # bugList = [('manybugs:python:69783-69784', 6000)]
+    # bugList = [('manybugs:python:69709-69710', 6027)]
+    # bugList = [('manybugs:python:69223-69224', 6027)]
+    # bugList = [('manybugs:php:2012-02-25-c1322d2505-cfa9c90b20', 6027)]
+
+
+    # parallelRun(fixTests,bugList)
+    # parallelRun(findTests,bugList,max_workers=1)
+    results = parallelRunMerge(validateCore, bugList)
+    # #
+    with open(join(DATA_PATH, 'mayBugsValidateNew'), 'w',
+              encoding='utf-8') as writeFile:
+        # if levelPatch == 0:
+        writeFile.write('\n'.join(results))
+    # print(results)
+
+def findTests(t):
+    bugName, port = t
+    preId, postId = bugName.split(':')[-1].split('-')[-2:]
+
+    preTests= join(DATA_PATH, 'manybugs', bugName,'test-' + preId + '.out')
+    postTests  = join(DATA_PATH, 'manybugs', bugName,'test-' + postId + '.out')
+
+    with open(preTests,mode='r') as preTestFile:
+        preTestResults = preTestFile.read()
+
+    with open(postTests,mode='r') as postTestFile:
+        postTestResults = postTestFile.read()
+
+    preTestResults
+    preFails = findAllTests(r"FAIL .*\[(.*)\]", preTestResults)
+    prePasses = findAllTests(r"PASS .*\[(.*)\]", preTestResults)
+    postFails = findAllTests(r"FAIL .*\[(.*)\]", postTestResults)
+    postPasses = findAllTests(r"PASS .*\[(.*)\]", postTestResults)
+    existingTests = get_filepaths(join(DATA_PATH, 'manybugs', bugName, 'src'), '.phpt')
+    existingTests = [i.split(join(DATA_PATH, 'manybugs', bugName, 'src/'))[-1] for i in existingTests]
+    postPasses
+    if( set(preFails).difference(set(postFails)) == set(postPasses).difference(set(prePasses))):
+        failingList = list(set(preFails).difference(set(postFails)))
+
+        passingList = list(set(postPasses).intersection(set(prePasses)))
+
+        failingList = [i for i in failingList if i in existingTests]
+        passingList = [i for i in passingList if i in existingTests]
+        if( len(failingList) > 0 and len(passingList)> 0):
+            with open(join(DATA_PATH, 'manybugs', bugName, 'failing.tests.txt'),mode='w') as failFile:
+                failFile.write('\n'.join(failingList))
+            with open(join(DATA_PATH, 'manybugs', bugName, 'passing.tests.txt'),mode='w') as passFile:
+                passFile.write('\n'.join(passingList))
+        else:
+            print('something wrong ' + bugName)
+    else:
+        print('something wrong ' + bugName)
+
+
+
+
+def findAllTests(regex,test_str):
+    matches = re.finditer(regex, test_str, re.MULTILINE)
+    res = []
+    for matchNum, match in enumerate(matches, start=1):
+
+        # print("Match {matchNum} was found at {start}-{end}: {match}".format(matchNum=matchNum, start=match.start(),
+        #                                                                     end=match.end(), match=match.group()))
+
+        for groupNum in range(0, len(match.groups())):
+            groupNum = groupNum + 1
+            res.append(match.group(groupNum))
+            # print("Group {groupNum} found at {start}-{end}: {group}".format(groupNum=groupNum,
+            #                                                                 start=match.start(groupNum),
+            #                                                                 end=match.end(groupNum),
+            #                                                                 group=match.group(groupNum)))
+    return res
+
+def patch_validate_mine():
+    logger = logging.getLogger()
+
+    for k,v in logger.manager.loggerDict.items():
+        if(k.startswith('bugzoo')):
+            if isinstance(v,logging.Logger):
+                v.setLevel(logging.ERROR)
+
+    buglist = listdir(join(DATA_PATH,'manybugs'))
+
+    # for i in range(0,len(bugList)):
+    bugList = []
+    port = 6000
+    for b in buglist:
+        if b== '.DS_Store':
+            continue
+        t = b, port
+        bugList.append(t)
+        if port == 6300:
+            port = 6000
+        port += 1
+    # bugList = [('manybugs:python:69783-69784', 6000)]
+    # bugList = [('manybugs:python:69709-69710', 6027)]
+    # bugList = [('manybugs:python:69223-69224', 6027)]
+    results = parallelRunMerge(myvalidateCore, bugList,max_workers=12)
+
+    with open(join(DATA_PATH, 'mayBugsPatchResults'), 'w',
+              encoding='utf-8') as writeFile:
+        # if levelPatch == 0:
+        writeFile.write('\n'.join(results))
+    print(results)
+
+
+
+from bugzoo import Patch, Client
+
+def test_all(bug, container,  client,validTests):
+    test_outcomes = {}  # type: Dict[TestCase, TestOutcome]
+    failure_cases = []
+    failure = []
+    total = len(bug.tests._tests)
+    for test in bug.tests:
+        # if test.name in validTests:
+        test_outcomes[test] = client.containers.test(container, test)
+        # if test.expected_outcome != test_outcomes[test].passed:
+        if test_outcomes[test].passed != True:
+            failure.append(test.name)
+            failure_cases.append(test.command)
+            break
+    return failure_cases, failure, total, test_outcomes
+
+
+def fixTests(t):
+    bugName, port = t
+    container = None
+    with bugzoo.server.ephemeral(port=port, verbose=False, timeout_connection=30000) as client:
+        try:
+
+            bug = client.bugs[bugName]
+            if client.bugs.is_installed(bug):
+                pass
+            else:
+                client.bugs.build(bug)
+
+            container = client.containers.provision(bug)
+            preId, postId = bugName.split(':')[-1].split('-')[-2:]
+
+            cmd = 'make distclean'
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+
+            cmd = 'git reset --hard && git clean -fd'
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+
+            cmd = 'git checkout "{}"'.format(preId)
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+            cmd ='cat ../libxml.patch | patch -p0'
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+
+            cmd = 'find . -name tests -type d -exec git checkout {} {{}} \;'.format(postId)
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+
+            cmd ='./buildconf'
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+            if preId in ['74343ca506', '8138f7de40']:
+                cmd ='./configure --disable-phar "CFLAGS=-fprofile-arcs -ftest-coverage" "CXXFLAGS=-fprofile-arcs -ftest-coverage" "LDFLAGS=-lgcov"'
+            else:
+                cmd = './configure "CFLAGS=-fprofile-arcs -ftest-coverage" "CXXFLAGS=-fprofile-arcs -ftest-coverage" "LDFLAGS=-lgcov"'
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+
+            cmd = 'make -j{}'.format(4)
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+
+            cmd = 'no | make test -s > ../test-'+preId+'.out'
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+            ###
+
+            cmd = 'make distclean'
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+
+            cmd = 'git reset --hard && git clean -fd'
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+
+            cmd = 'git checkout "{}"'.format(postId)
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+            cmd = 'cat ../libxml.patch | patch -p0'
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+
+            cmd = './buildconf'
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+            if preId in ['74343ca506', '8138f7de40']:
+                cmd = './configure --disable-phar "CFLAGS=-fprofile-arcs -ftest-coverage" "CXXFLAGS=-fprofile-arcs -ftest-coverage" "LDFLAGS=-lgcov"'
+            else:
+                cmd = './configure "CFLAGS=-fprofile-arcs -ftest-coverage" "CXXFLAGS=-fprofile-arcs -ftest-coverage" "LDFLAGS=-lgcov"'
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+
+            cmd = 'make -j{}'.format(4)
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+
+            cmd = 'no | make test -s > ../test-' + postId + '.out'
+            o = client.containers.exec(container=container, command=cmd, context='/experiment/src')
+
+            cmd = 'docker cp ' + container.id +':/experiment/test-'+preId+'.out ' + join(DATA_PATH,'manybugs',bugName,'test-'+preId+'.out')
+            o,e = shellGitCheckout(cmd)
+            print(o, e)
+            cmd = 'docker cp ' + container.id +':/experiment/test-'+postId+'.out ' +join(DATA_PATH,'manybugs',bugName,'test-'+postId+'.out')
+            o,e = shellGitCheckout(cmd)
+
+            print(o,e)
+
+
+
+
+        except Exception as e:
+            logging.error(e)
+            # continue
+        finally:
+            # ''
+            cmd = 'docker stop {}'.format(container.id)
+            out, e = shellGitCheckout(cmd)
+            # docker stop $(docker ps -q)
+
+if __name__ == "__main__":
+    patch_validate()

python/patch_validate_introClass.py

@@ -0,0 +1,125 @@
+import bugzoo
+from bugzoo import server, Container
+import csv
+import os
+
+introClassFile = './data/introClassData.txt'
+
+def patch_validate():
+    with bugzoo.server.ephemeral(port=8081,verbose=True,timeout_connection=3000) as client:
+        # url = "http://127.0.0.1:6060"
+        # client = bugzoo.Client(url)
+        # bug = client.bugs['introclass:checksum:08c7ea:006']
+        bugList = []
+        with open(introClassFile,'r') as file:
+            for line in file.readlines():
+                bugList.append(line.strip())
+
+        for i in range(0,len(bugList)):
+            try:
+                bugName = bugList[i]
+                # if bugName != 'introclass:digits:070455:000':
+                #     continue
+                print("bugName: {}".format(bugName),end=' ')
+                bug = client.bugs[bugName]
+                if client.bugs.is_installed(bug):
+                    # print("the image is installed! :-)")
+                    pass
+                else:
+                    client.bugs.build(bug)
+                    # client.bugs.download(bug)
+                    # print("the image is not installed :'(")
+
+                # print("creating container...")
+                container = client.containers.provision(bug)
+                # print("container is ready")
+
+                print("First_test:",end=' ')
+                pre_test_outcomes = {}
+                pre_failure_cases, pre_failure, total, pre_test_outcomes = test_all(bug, container, client)
+                print("@fail:{}@total:{}".format(pre_failure, total),end=' ')
+                print("@pre_failure_cases:{}".format(pre_failure_cases),end=' ')
+
+                # print("patching...")
+                diff_files = get_diff(bug.name, client, container)
+                patch_result = patch_application(client, container, diff_files)
+                if not patch_result:
+                    print("@{}@".format(patch_result))
+                    continue
+                print("@{}@".format(patch_result), end=' ')
+
+                print("Second_test:",end=' ')
+                post_test_outcomes = {}
+                post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client)
+                print("@fail:{}@total:{}".format(post_failure, total),end=' ')
+                print("@post_failure_cases:{}".format(post_failure_cases))
+            except:
+                print(" ")
+                continue
+        pass
+
+from bugzoo import Patch, Client
+
+def test_all(bug, container,  client):
+    test_outcomes = {}  # type: Dict[TestCase, TestOutcome]
+    failure_cases = []
+    failure = 0
+    total = len(bug.tests._tests)
+    for test in bug.tests:
+        test_outcomes[test] = client.containers.test(container, test)
+        # if test.expected_outcome != test_outcomes[test].passed:
+        if test_outcomes[test].passed != True:
+            failure += 1
+            failure_cases.append(test.command)
+            break
+    return failure_cases, failure, total, test_outcomes
+
+def get_diff(bug, client, container):
+    patch_list = []
+    path = './data/introclass/' + bug
+    for root, dirs, files in os.walk(path):
+        for name in files:
+            if name.endswith('.patch'):
+                patch_list.append(os.path.join(root, name))
+
+    diff_files = []
+    for patch_file in patch_list:
+        unidiff = ''
+        with open(patch_file,'r') as file:
+            for line in file.readlines():
+                if line.startswith('\\'):
+                    continue
+                elif line.startswith('---'):
+                    unidiff += '--- ' + bug.split(':')[1] + '.c\n'
+                elif line.startswith('+++'):
+                    unidiff += '+++ ' + bug.split(':')[1] + '.c\n'
+                else:
+                    unidiff += line
+            diff_files.append(unidiff.strip())
+
+    return diff_files
+
+
+
+def patch_application(client: Client, container: Container, diff_files: list) -> None:
+    result = None
+    for unidiff in diff_files:
+        if len(unidiff) < 5:
+            continue
+        # first, we build a Patch object using a unified-format diff
+        patch = Patch.from_unidiff(unidiff)
+
+        # we then attempt to apply the patch to the source code
+        re = client.containers.patch(container, patch)
+        if not re:
+            return False
+        else:
+            result = True
+    # finally, we rebuild the program inside the container
+    client.containers.build(container)
+
+    return result
+
+
+if __name__ == "__main__":
+    patch_validate()

python/patch_validate_introClass2.py

@@ -0,0 +1,145 @@
+import bugzoo
+from bugzoo import server, Container
+from common.commons import *
+
+from common.commons import shellGitCheckout
+DATA_PATH = os.environ["DATA_PATH"]
+introClassFile = join(DATA_PATH,'introClassData.txt')
+
+def patch_validate():
+    with bugzoo.server.ephemeral(port=8081,verbose=True,timeout_connection=3000) as client:
+        # url = "http://127.0.0.1:6060"
+        # client = bugzoo.Client(url)
+        # bug = client.bugs['introclass:checksum:08c7ea:006']
+        bugList = []
+        with open(introClassFile,'r') as file:
+            for line in file.readlines():
+                bugList.append(line.strip())
+
+        for i in range(0,len(bugList)):
+            try:
+                bugName = bugList[i]
+                # if bugName != 'introclass:digits:070455:000':
+                #     continue
+                print("bugName: {}".format(bugName),end=' ')
+                bug = client.bugs[bugName]
+                if client.bugs.is_installed(bug):
+                    # print("the image is installed! :-)")
+                    pass
+                else:
+                    client.bugs.build(bug)
+                    # client.bugs.download(bug)
+                    # print("the image is not installed :'(")
+
+                # print("creating container...")
+                container = client.containers.provision(bug)
+                # print("container is ready")
+
+                print("First_test:",end=' ')
+                pre_test_outcomes = {}
+                pre_failure_cases, pre_failure, total, pre_test_outcomes = test_all(bug, container, client)
+                print("@fail:{}@total:{}".format(pre_failure, total),end=' ')
+                print("@pre_failure_cases:{}".format(pre_failure_cases),end=' ')
+
+                # print("patching...")
+                path = os.path.abspath('data') + '/introclass_patched/' + bugName
+                patch_path = path + '/patches'
+                avaliable_patch = os.path.abspath('data') + '/introclass2/' + bugName + '/' + 'patches'
+                patch_names = os.listdir(patch_path)
+                for patch_name in patch_names:
+                    # if patch_name not in os.listdir(avaliable_patch):
+                    #     continue
+                    patch = patch_path + '/' + patch_name
+                    patch_result = patched_application(path, bug.name, patch, client, container)
+                    if patch_result == -1 or patch_result.code != 0:
+                        print("@{}@".format('False'), end='')
+                        print("{}".format('F'), end=' ')
+                        continue
+                    print("@{}@".format('True'), end='')
+
+                    print("Second_test:",end=' ')
+                    post_test_outcomes = {}
+                    post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client)
+                    print("{}".format(post_failure), end=' ')
+                    if post_failure == 0:
+                        print("fix {} by {}".format(bugName, patch_name))
+                    # print("@fail:{}@total:{}".format(post_failure, total),end=' ')
+                    # print("@post_failure_cases:{}".format(post_failure_cases))
+
+                cmd = 'docker rm -fv {}'.format(container.id)
+                output, e = shellGitCheckout(cmd)
+                print(" ")
+            except:
+                print(" ")
+                continue
+        pass
+
+from bugzoo import Patch, Client
+
+def test_all(bug, container,  client):
+    test_outcomes = {}  # type: Dict[TestCase, TestOutcome]
+    failure_cases = []
+    failure = 0
+    total = len(bug.tests._tests)
+    for test in bug.tests:
+        test_outcomes[test] = client.containers.test(container, test)
+        # if test.expected_outcome != test_outcomes[test].passed:
+        if test_outcomes[test].passed != True:
+            failure += 1
+            failure_cases.append(test.command)
+            break
+    return failure_cases, failure, total, test_outcomes
+
+def patch_application(path, bugName, patch, client: Client, container: Container):
+    buggroup = bugName.split(":")[1]
+    # path = join(BUGDIR,bug)
+    program = path + '/' + buggroup + '.c'
+    fixedFile = path + '/oracle.c'
+    patchPath = patch
+    with open(patch, 'r+', encoding='utf-8') as file:
+        result = ''
+        for line in file:
+            if line.startswith('---'):
+                # result += '--- ' + program + '\n'
+                result += '--- ' + buggroup + '.c' + '\n'
+            elif line.startswith('+++'):
+                # result += '+++ ' + fixedFile + '\n'
+                result += '+++ ' + 'oracle.c' + '\n'
+            else:
+                result += line
+    with open(patch, 'w', encoding='utf-8') as file:
+        file.writelines(result)
+
+    cmd = 'rm  ' + fixedFile
+    output, e = shellGitCheckout(cmd)
+
+    cmd = 'patch --ignore-whitespace -p0 < ' + '\"' + patchPath + '\"'
+    # cmd = 'patch  -p0 < ' + '\"' + patchPath + '\"'
+    output, e = shellGitCheckout(cmd)
+
+    if os.path.exists(fixedFile):
+        cmd = 'docker cp ' + fixedFile + ' ' + container.id + ':/experiment/'
+        output, e = shellGitCheckout(cmd)
+
+        cmd = 'rm .genprog_test_cache.json & gcc -o {} {}.c'.format(buggroup, 'oracle')
+        output = client.containers.exec(container=container, command=cmd, context='/experiment/')
+        return output
+    else:
+        return -1
+
+def patched_application(path, bugName, patched, client: Client, container: Container):
+    buggroup = bugName.split(":")[1]
+    # path = join(BUGDIR,bug)
+    program = path + '/' + buggroup + '.c'
+    fixedFile = patched.split('/')[-1]
+
+    cmd = 'docker cp ' + patched + ' ' + container.id + ':/experiment/'
+    output, e = shellGitCheckout(cmd)
+
+    cmd = 'rm .genprog_test_cache.json & mv {} {}.c & gcc -o {} {}'.format(fixedFile,fixedFile, buggroup, fixedFile+'.c')
+    output = client.containers.exec(container=container, command=cmd, context='/experiment/')
+    return output
+
+
+if __name__ == "__main__":
+    patch_validate()

python/sprinferIndex.py

@@ -0,0 +1,454 @@
+from common.commons import *
+DATA_PATH = os.environ["DATA_PATH"]
+SPINFER_PATH = os.environ["spinfer"]
+
+SPINFER_INDEX_PATH = os.environ["dataset"]
+COCCI_PATH = join(os.environ["coccinelle"],'spatch')
+DATASET = os.environ["dataset"]
+
+
+def indexCore():
+
+    print(SPINFER_INDEX_PATH)
+    # dbDir = join(DATA_PATH, 'redis')
+    # startDB(dbDir, "6399", "ALLdumps-gumInput.rdb")
+    # import redis
+    # redis_db = redis.StrictRedis(host="localhost", port=6399, db=0)
+    # keys = redis_db.hkeys("dump")#hkeys "dump"
+    # # keys = redis_db.scan(0, match='*', count='1000000')
+    #
+    # matches = pd.DataFrame(keys, columns=['pairs_key'])
+    #
+    # # matches = load_zipped_pickle(join(DATA_PATH,'singleHunks'))
+    # matches['pairs_key'] = matches['pairs_key'].apply(lambda x: x.decode())
+    # matches['root'] = matches['pairs_key'].apply(lambda x: x.split('/')[0])
+    # matches['size'] = matches['pairs_key'].apply(lambda x: x.split('/')[1])
+    # matches['file'] = matches['pairs_key'].apply(lambda x: x.split('/')[2])
+    # matches['hunk'] = matches['pairs_key'].apply(lambda x: x.split('/')[2].split('_')[-1])
+    # matches['fileName'] = matches['pairs_key'].apply(lambda x: '_'.join(x.split('/')[2].split('_')[:-1]))
+    # test = matches[['fileName', 'hunk']]
+    # df = test.groupby(by=['fileName'], as_index=False).agg(lambda x: x.tolist())
+    # # sDF = df[df.hunk.apply(lambda x: True if x == ['0'] else False)]
+    # sDF = df[df.hunk.apply(lambda x: True if len(x)<10005 else False)]
+    # singleHunkedFiles = sDF.fileName.unique().tolist()
+    # singleHunkedFiles = [i.replace('.txt', '') for i in singleHunkedFiles]
+
+    clusterPath = join(DATA_PATH, 'shapes')
+    roots = listdir(clusterPath)
+    roots = [i for i in roots if not (i.startswith('.') or i.endswith('.pickle'))]
+
+
+    for root in roots:
+        root
+        sizes = listdir(join(clusterPath,root))
+        sizes = [f for f in sizes if not f.startswith('.')]
+        for size in sizes:
+
+            # actions = listdir(join(clusterPath,root,size))
+            # for action in actions:
+                clusters = listdir(join(clusterPath,root,size))
+                clusters = [f for f in clusters if not f.startswith('.')]
+                for cluster in clusters:
+                    members = listdir(join(clusterPath,root,size,cluster))
+                    members= [f for f in members if not f.startswith('.')]
+                    # members = [re.sub('^linux_','',i) for i in members]
+
+                    # csMembers = {}
+                    # for j in [i.split('.txt_') for i in members]:
+                    #     k,v = j
+                    #     k = k +'.txt'
+                    #     if k in csMembers:
+                    #         tmp = csMembers[k]
+                    #         tmp.append(v)
+                    #         csMembers[k] = tmp
+                    #     else:
+                    #         csMembers[k] = [v]
+                    #
+                    # splitMembers = []
+                    # for k,v in csMembers.items():
+                    #     hunks = df[df.fileName == k].iloc[0].hunk
+                    #     if set(hunks) == set(v):
+                    #         splitMembers.append(k)
+
+
+                    members = [re.sub('\.txt\_\d+','',i) for i in members]
+                    members = list(set(members))
+                    # members = [i for i in members if i in singleHunkedFiles or i in splitMembers]
+                    if len(members) > 1:
+                            lines = []
+                            for member in members:
+                                split =member.split('_')
+                                pj = split[0]
+                                member ='_'.join(split[1:])
+                                line = pj+'/prevFiles/prev_' + member + ' ' + pj+'/revFiles/' + member + '\n'
+                                lines.append(line)
+
+                            if len(lines)>0:
+                                with open(join(SPINFER_INDEX_PATH, root+"_"+size +'_'+cluster+'.index'), 'w', encoding='utf-8') as writeFile:
+                                    # if levelPatch == 0:
+                                    writeFile.write(''.join(lines))
+
+
+
+def test():
+    indexes = listdir(SPINFER_INDEX_PATH)
+    indexes = [i for i in indexes if i.endswith('.index')]
+    coccis = listdir(join(SPINFER_INDEX_PATH,'cocci'))
+
+    if not os.path.exists(join(SPINFER_INDEX_PATH,'indexNC')):
+        os.mkdir(join(SPINFER_INDEX_PATH,'indexNC'))
+
+    for i in indexes:
+        if re.sub('\.index', '.cocci', i) not in coccis:
+            shutil.move(join(SPINFER_INDEX_PATH,i),join(SPINFER_INDEX_PATH,'indexNC',i))
+
+
+def runSpinfer():
+    indexCore()
+    indexes = listdir(SPINFER_INDEX_PATH)
+    indexes = [i for i in indexes if i.endswith('.index')]
+    # indexes = ['if_9_44.index']
+    if not os.path.exists(join(SPINFER_INDEX_PATH,'cocci')):
+        os.mkdir(join(SPINFER_INDEX_PATH,'cocci'))
+    os.chdir(SPINFER_INDEX_PATH)
+    pairs = []
+    for i in indexes:
+        pairs.append((os.path.getsize(i),i))
+    pairs.sort(key=lambda s:s[0])
+
+    coccis = listdir(join(SPINFER_INDEX_PATH,'cocci'))
+
+    cmdList = []
+    bigCmdList = []
+    for t in pairs:
+        sizes, idx = t
+        cocciName = re.sub('\.index', '.cocci', idx)
+        if cocciName in coccis:
+            continue
+        # cmd = SPINFER_PATH + " --no-progress -f " + idx + " -o cocci/" + cocciName
+        # bigCmdList.append(cmd)
+        if sizes < 500:
+        #     #cmd = SPINFER_PATH + " -j 16 -f " + idx + " -o cocci/" + re.sub('\.index', '.cocci', idx)
+        #     # cmd = SPINFER_PATH + " --no-progress --genericity 1 -j 16 -f " + idx + " -o cocci/" + cocciName
+            cmd = SPINFER_PATH + " --no-progress -f " + idx + " -o cocci/" + cocciName
+            cmdList.append(cmd)
+        else:
+        #     # cmd = SPINFER_PATH + " --no-progress --genericity 1 -j 1 -f " + idx + " -o cocci/" + cocciName
+            cmd = SPINFER_PATH + " --no-progress -f " + idx + " -o cocci/" + cocciName
+            bigCmdList.append(cmd)
+
+    # for cmd in cmdList:
+    #     logging.info(cmd)
+    #     output, e = shellGitCheckout(cmd)
+    #     logging.info(output)
+    parallelRun(callSpinfer,cmdList)
+    # logging.info('big commands')
+    # for cmd in bigCmdList:
+    #     #     logging.info(cmd)
+    #     #     output, e = shellGitCheckout(cmd)
+    #     #     logging.info(output)
+    parallelRun(callSpinfer,bigCmdList)
+
+    # if not os.path.exists(join(DATA_PATH,'cocci')):
+    #     os.mkdir(join(DATA_PATH,'cocci'))
+
+
+        # logging.info(cmd)
+        # output,e = shellGitCheckout(cmd)
+        # logging.info(output)
+
+def divideCoccis():
+    import datetime
+    shutil.copytree(join(SPINFER_INDEX_PATH,'cocci'),join(DATA_PATH,'cocci'+ datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')))
+
+    coccis =os.listdir(join(SPINFER_INDEX_PATH,'cocci'))
+    for cocci in coccis:
+        with open(join(SPINFER_INDEX_PATH,'cocci',cocci), 'r') as iFile:
+            idx = iFile.readlines()
+            idx
+        values = np.array(idx)
+        points = np.where(values == '@@\n')
+
+        points = list(itertools.chain.from_iterable(points))
+        if len(points) == 0:
+            os.remove(join(SPINFER_INDEX_PATH, 'cocci', cocci))
+        # patches = list(zip(*([iter(points)] * 2)))
+        patches = list(pairwise(points[::2]))# every second element in list
+        if len(patches) > 0:
+            i = 0;
+            for t in patches :
+                t
+                with open(join(SPINFER_INDEX_PATH, 'cocci', cocci+str(i)), 'w') as iFile:
+                    iFile.writelines(idx[t[0]:t[1]])
+                i=i+1
+            t
+            with open(join(SPINFER_INDEX_PATH, 'cocci', cocci + str(i)), 'w') as iFile:
+                iFile.writelines(idx[t[1]:])
+            os.remove(join(SPINFER_INDEX_PATH, 'cocci', cocci))
+
+def removeDuplicates():
+    commentPattern = r"(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)"
+    coccis =os.listdir(join(SPINFER_INDEX_PATH, 'cocci'))
+    cocciPatterns = pd.DataFrame(columns=['cid', 'pattern','inferedFrom'])
+    ind = 0
+    for cocci in coccis:
+        with open(join(SPINFER_INDEX_PATH, 'cocci', cocci), 'r') as iFile:
+            idx = iFile.read()
+            idx
+            inferedFrom = re.search(r"// Infered from:(.*)\n",idx).groups()
+            pattern = re.sub(commentPattern, '', idx, re.DOTALL)
+            cocciPatterns.loc[ind] = [cocci,pattern,inferedFrom]
+            ind = ind +1
+    cocciPatterns['iFiles'] = cocciPatterns.inferedFrom.apply(lambda x: getInferred(x[0]))
+
+
+    save_zipped_pickle(cocciPatterns,join(DATA_PATH,'allCocciPatterns.pickle'))
+    allPatterns = cocciPatterns.cid.values.tolist()
+    uniquePatterns = cocciPatterns.drop_duplicates(subset=['pattern']).cid.values.tolist()
+    toRemove = list(set(allPatterns).difference(uniquePatterns))
+    print(toRemove)
+    for p in toRemove:
+        os.remove(join(SPINFER_INDEX_PATH, 'cocci', p))
+    print(len(uniquePatterns))
+
+
+def filterPatterns():
+    cocciPatterns = load_zipped_pickle(join(DATA_PATH,'allCocciPatterns.pickle'))
+    cocciPatterns['filtered'] = cocciPatterns.iFiles.apply(lambda x: [i for i in x if  (
+                i.startswith('(libtiff') or i.startswith('(php-src') or i.startswith('(cpython'))])
+    return cocciPatterns[cocciPatterns.filtered.str.len() > 0].cid.values.tolist()
+
+def getInferred(x):
+    regex = r"\(.*?\)"
+    matches = re.finditer(regex, x)
+
+    results = []
+    for matchNum, match in enumerate(matches, start=1):
+        results.append(match.group())
+    return results
+
+
+def getNegLines(x):
+    negLines = r"^-(.*)"
+    matches = re.finditer(negLines, x, re.MULTILINE)
+    res = []
+    for matchNum, match in enumerate(matches, start=1):
+
+        for groupNum in range(0, len(match.groups())):
+            groupNum = groupNum + 1
+
+            # print("Group {groupNum} found at {start}-{end}: {group}".format(groupNum=groupNum,
+            #                                                                 start=match.start(groupNum),
+            #                                                                 end=match.end(groupNum),
+            #                                                                 group=match.group(groupNum)))
+            res.append(match.group(groupNum))
+    from common.preprocessing import getTokensForPatterns
+    tokens = getTokensForPatterns(res)
+
+    return tokens
+def patternOperations():
+    cocciPatterns = load_zipped_pickle(join(DATA_PATH, 'allCocciPatterns.pickle'))
+    # cocciPatterns.pattern.apply(lambda x:re.search(r"@@(.*)@@",x,re.DOTALL|re.M).groups())
+    cocciPatterns['patternTokens'] = cocciPatterns.pattern.apply(lambda x:getNegLines(x))
+    #set(getTokensForPatterns(lines)).intersection(set(tokens))
+    save_zipped_pickle(cocciPatterns,join(DATA_PATH, 'allCocciPatterns.pickle'))
+# import html
+# def getTokensForPatterns(res):
+#     if isinstance(res, list):
+#         merged = str()
+#         for r in res:
+#             if isinstance(r, list):
+#                 merged = merged + ' ' + ' '.join(r)
+#             else:
+#                 merged = merged +' ' + r
+#     else:
+#         merged=res
+#
+#     res = html.unescape(merged)
+#
+#     tokens = getTokens(res,False)
+#
+#     stripped = []
+#     for t in tokens:
+#         splits = re.split('\.|\(|\)|:|>|<|:|=|/|\\\\|\'|-|,|\]|\[|}|{|;',t)
+#         for s in splits:
+#             stripped.append(s)
+#     non_empty = [i for i in stripped if i != '']
+#     return non_empty
+# from nltk.tokenize import RegexpTokenizer
+#
+# def getTokens(re,printDetail=False):
+#     tokenizer = RegexpTokenizer(r'\S+')
+#     tokens = tokenizer.tokenize(re)
+#     if printDetail:
+#         print('=====TOKENS=========')
+#         print(tokens)
+#
+#     return tokens
+
+def filterCore(t):
+    # cocciPatterns = load_zipped_pickle(join(DATA_PATH, 'allCocciPatterns.pickle'))
+    src,spfile = t
+    srcPath = src
+    patchName = src.split('/')[-1]
+    manybug = src.split('/')[-2]
+    # with open(srcPath, mode='r') as srcFile:
+    #     lines = srcFile.read()
+    # scTokens = getTokensForPatterns(lines)
+
+    # tokens = cocciPatterns[cocciPatterns.cid == spfile].iloc[0].patternTokens
+    # patternFilter = set(scTokens).intersection(set(tokens))
+    # if len(patternFilter) > 0:
+    cmd = COCCI_PATH + ' --sp-file ' + join(DATASET, 'cocci', spfile) + ' ' + srcPath + ' --patch -o' + join(
+        DATA_PATH, "introclass", manybug, 'patches', patchName) + ' > ' + join(DATA_PATH, "introclass", manybug,
+                                                                               'patches',
+                                                                               patchName + spfile + '.txt')
+    # cmd = COCCI_PATH + ' --sp-file ' + join(DATASET,'cocci',spfile) + ' ' + srcPath + ' -o ' + join(DATA_PATH,"introclass",manybug,'patches',patchName+spfile+'.txt')
+    t = cmd, manybug, patchName, spfile, srcPath
+    return t
+    # return None
+
+
+def patchCoreIntro():
+
+
+    manybugs = listdir(join(DATA_PATH,"introclass"))
+    spfiles = listdir(join(DATASET,'cocci'))
+    # workList = []
+
+
+
+    filterList =[]
+    for manybug in manybugs:
+        # files = listdir(join(join(DATA_PATH,"manybugs",manybug,'diffs')))
+        if  os.path.exists(join(DATA_PATH, "introclass", manybug, 'patches')):
+            shutil.rmtree(join(DATA_PATH, "introclass", manybug, 'patches'))
+
+            os.mkdir(join(DATA_PATH, "introclass", manybug, 'patches'))
+        else:
+            os.mkdir(join(DATA_PATH, "introclass", manybug, 'patches'))
+
+        if  os.path.exists(join(DATA_PATH, "introclass", manybug, 'patched')):
+            shutil.rmtree(join(DATA_PATH, "introclass", manybug, 'patched'))
+
+            os.mkdir(join(DATA_PATH, "introclass", manybug, 'patched'))
+        else:
+            os.mkdir(join(DATA_PATH, "introclass", manybug, 'patched'))
+
+        files = get_filepaths(join(DATA_PATH, "introclass", manybug), '.c')
+        sources = [i for i in files if not (i.endswith('oracle.c.patch') or i.endswith('oracle.c'))]
+
+        filterList.extend(list(itertools.product(sources, spfiles)))
+    print(len(filterList))
+    workList = parallelRunMerge(filterCore,filterList)
+
+        # for src in sources:
+        #     # srcPath = src.replace('/diffs/','/src/')
+        #     srcPath = src
+        #
+        #
+        #     with open(srcPath, mode='r') as srcFile:
+        #         lines = srcFile.read()
+        #     scTokens = getTokensForPatterns(lines)
+        #
+        #     patchName = src.split('/')[-1]
+        #     for spfile in spfiles:
+        #          tokens = cocciPatterns[cocciPatterns.cid == spfile].iloc[0].patternTokens
+        #          patternFilter = set(scTokens).intersection(set(tokens))
+        #          if len(patternFilter) > 0:
+        #              cmd = COCCI_PATH + ' --sp-file ' + join(DATASET,'cocci',spfile) + ' ' + srcPath + ' --patch -o' + join(DATA_PATH,"introclass",manybug,'patches',patchName) + ' > ' + join(DATA_PATH, "introclass", manybug, 'patches', patchName + spfile + '.txt')
+        #              # cmd = COCCI_PATH + ' --sp-file ' + join(DATASET,'cocci',spfile) + ' ' + srcPath + ' -o ' + join(DATA_PATH,"introclass",manybug,'patches',patchName+spfile+'.txt')
+        #              t = cmd,manybug,patchName,spfile,srcPath
+        #              workList.append(t)
+        #          else:
+        #             logging.info("Skip "+ srcPath + " " + spfile)
+    workList = list(filter(None, workList))
+    print(len(workList))
+    # for l in workList:
+    #     cocciCore2(l)
+    parallelRun(cocciCore2,workList)
+
+def patched():
+    manybugs = listdir(join(DATA_PATH,"introclass"))
+    # spfiles = listdir(join(DATASET,'cocci'))
+    workList = []
+    for manybug in manybugs:
+        # files = listdir(join(join(DATA_PATH,"manybugs",manybug,'diffs')))
+        if  os.path.exists(join(DATA_PATH, "introclass_patched", manybug, 'patches')):
+            shutil.rmtree(join(DATA_PATH, "introclass_patched", manybug, 'patches'))
+
+            os.makedirs(join(DATA_PATH, "introclass_patched", manybug, 'patches'))
+        else:
+            os.makedirs(join(DATA_PATH, "introclass_patched", manybug, 'patches'))
+        files = get_filepaths(join(DATA_PATH, "introclass", manybug), '.c')
+        sources = [i for i in files if not (i.endswith('oracle.c.patch') or i.endswith('oracle.c'))]
+
+
+        for src in sources:
+            # srcPath = src.replace('/diffs/','/src/')
+            srcPath = src
+            patchName = src.split('/')[-1]
+
+
+            spfiles = listdir(join(DATA_PATH,"introclass",manybug,'patches'))
+            for spfile in spfiles:
+                 spfile =spfile.replace(patchName,'').replace('.txt','')
+                 cmd = COCCI_PATH + ' --sp-file ' + join(DATASET,'cocci',spfile) + ' ' + srcPath + ' -o ' + join(DATA_PATH,"introclass_patched",manybug,'patches',patchName+spfile+'.c')
+                 t = cmd,manybug,patchName,spfile
+                 workList.append(t)
+    parallelRun(cocciCore,workList)
+
+
+
+def cocciCore2(t):
+    cmd,manybug,patchName,spfile,srcPath = t
+    # cocciPatterns = load_zipped_pickle(join(DATA_PATH, 'allCocciPatterns.pickle'))
+    # tokens = cocciPatterns[cocciPatterns.cid == spfile].iloc[0].patternTokens
+    # with open(srcPath, mode='r') as srcFile:
+    #     lines = srcFile.read()
+
+    # patternFilter = set(getTokensForPatterns(lines)).intersection(set(tokens))
+    # if len(patternFilter) > 0:
+
+    # logging.info(cmd)
+    output, e = shellGitCheckout(cmd)
+    # logging.info(output)
+    patchSize = os.path.getsize(join(DATA_PATH,"introclass",manybug,'patches',patchName+spfile+'.txt'))
+    if patchSize == 0 :
+        os.remove(join(DATA_PATH,"introclass",manybug,'patches',patchName+spfile+'.txt'))
+    else:
+        cmd = 'patch -d '+join(DATA_PATH, "introclass", manybug)+' -i '+join(DATA_PATH,"introclass",manybug,'patches',patchName+spfile+'.txt')+' -o '+join(DATA_PATH,"introclass",manybug,'patched',patchName+spfile+'.c')
+        o,e = shellGitCheckout(cmd)
+
+def cocciCore(t):
+    cmd, manybug, patchName, spfile = t
+    # logging.info(cmd)
+    output, e = shellGitCheckout(cmd)
+    # logging.info(output)
+    patchSize = os.path.getsize(join(DATA_PATH,"introclass",manybug,'patches',patchName+spfile+'.txt'))
+    if patchSize == 0 :
+     os.remove(join(DATA_PATH,"introclass",manybug,'patches',patchName+spfile+'.txt'))
+# def patchCore():
+#     manybugs = listdir(join(DATA_PATH,"manybugs"))
+#     spfiles = listdir(join(DATASET,'cocci'))
+#     for manybug in manybugs:
+#         # files = listdir(join(join(DATA_PATH,"manybugs",manybug,'diffs')))
+#         if not os.path.exists(join(DATA_PATH, "manybugs", manybug, 'patches')):
+#             os.mkdir(join(DATA_PATH, "manybugs", manybug, 'patches'))
+#         if not os.path.exists(join(DATA_PATH, "manybugs", manybug, 'patched')):
+#             os.mkdir(join(DATA_PATH, "manybugs", manybug, 'patched'))
+#
+#         files = get_filepaths(join(DATA_PATH,"manybugs",manybug,'diffs'),'.patch')
+#         sources = [i.replace('.c.patch','.c') for i in files if i.endswith('.c.patch')]
+#         for src in sources:
+#             srcPath = src.replace('/diffs/','/src/')
+#             patchName = src.split('/')[-1]
+#             for spfile in spfiles:
+#                  cmd = COCCI_PATH + ' -j 16 --sp-file ' + join(DATASET,'cocci',spfile) + ' ' + srcPath + ' -o ' + join(DATA_PATH,"manybugs",manybug,'patches',patchName+spfile+'.txt')
+#                  logging.info(cmd)
+#                  output, e = shellGitCheckout(cmd)
+#                  logging.info(output)
+#                  patchSize = os.path.getsize(join(DATA_PATH,"manybugs",manybug,'patches',patchName+spfile+'.txt'))
+#                  if patchSize == 0 :
+#                      os.remove(join(DATA_PATH,"manybugs",manybug,'patches',patchName+spfile+'.txt'))
+

python/stats.py

@@ -0,0 +1,572 @@
+from common.commons import *
+DATA_PATH = os.environ["DATA_PATH"]
+PROJECT_TYPE = os.environ["PROJECT_TYPE"]
+
+def statsNormal(isFixminer=True):
+    # tokens = join(DATA_PATH, 'tokens')
+    # actions = join(DATA_PATH, 'actions')
+    import redis
+    redis_db = redis.StrictRedis(host="localhost", port=6399, db=0)
+    # keys = redis_db.scan(0, match='*', count='1000000')
+    keys = redis_db.hkeys("dump")  # hkeys "dump"
+    matches = pd.DataFrame(keys, columns=['pairs_key'])
+
+    # matches = load_zipped_pickle(join(DATA_PATH,'singleHunks'))
+    matches['pairs_key'] = matches['pairs_key'].apply(lambda x: x.decode())
+    matches['root'] = matches['pairs_key'].apply(lambda x: x.split('/')[0])
+    matches['size'] = matches['pairs_key'].apply(lambda x: x.split('/')[1])
+    matches['file'] = matches['pairs_key'].apply(lambda x: x.split('/')[2])
+    matches['repo'] = matches['file'].apply(lambda x: x.split('_')[0])
+    matches['commit'] = matches['file'].apply(lambda x: x.split('_')[1])
+    matches['hunk'] = matches['pairs_key'].apply(lambda x: x.split('/')[2].split('_')[-1])
+    matches['fileName'] = matches['pairs_key'].apply(lambda x: '_'.join(x.split('/')[2].split('_')[:-1]))
+    test = matches[['fileName','hunk']]
+    df = test.groupby(by=['fileName'], as_index=False).agg(lambda x: x.tolist())
+
+    # df = load_zipped_pickle('datasetGroupByHunk.pickle')
+    yList = []
+    colNames = []
+
+    def haveAll(x):
+        check = dict()
+        if len(x)<2:
+            return False
+        for hunk in x:
+            filename, hunkNo = hunk.rsplit('_',1)
+            if filename in check:
+                check[filename].append(hunkNo)
+            else:
+                check[filename] = [hunkNo]
+
+        tv = []
+        for k,v in check.items():
+            selected = df[df.fileName == k]
+            if tuple(selected.iloc[0].hunk) == tuple(v):
+                tv.append(True)
+            else:
+                tv.append(False)
+        result = np.all(tv)
+        return result
+
+    def combinationOfPatterns(x):
+        check = dict()
+        if len(x)<2:
+            return False
+        for hunk in x:
+            filename, hunkNo = hunk.rsplit('_',1)
+            if filename in check:
+                check[filename].append(hunkNo)
+            else:
+                check[filename] = [hunkNo]
+
+        tv = []
+
+        res = check.copy()
+        for k,v in check.items():
+            # if len(v) == 1:
+            #     res.pop(k,None)
+            selected = df[df.fileName == k]
+            if set(selected.iloc[0].hunk) != set(v):
+                res.pop(k, None)
+        # result = np.all(tv)
+        indices = [i for i, val in enumerate(tv) if val == True]
+        return res
+    allHunks = []
+    def test(x):
+        try:
+            fileList = []
+            for i in x:
+
+                fileName = i.rsplit('_', 1)[0]
+
+                if fileName in byCombi.keys():
+                    fileList.append(i)
+            if len(fileList) > 0:
+
+                if (set(fileList).issubset(set(x))):
+                    # logging.info("In the same pattern")
+                    allHunks.extend(fileList)
+                    return True
+                else:
+                  logging.info("Mix")
+            return False
+        except Exception as e:
+            logging.error(e)
+
+
+    # for type in ['tokens', 'actions', 'shapes']:
+    for type in ['shapes']:
+        statsS,clusterDF = stats(type,isFixminer)
+        if isFixminer:
+            clusterDF = clusterDF[clusterDF.members.str.len() > 1]
+
+            clusterDF['vertical'] = clusterDF.members.apply(lambda x:len(set([i.rsplit('_',1)[0] for i in x])) == 1)
+            clusterDF['v-h'] = clusterDF.members.apply(lambda x: len(set([i.rsplit('_', 1)[0] for i in x])))
+            clusterDF['ms'] = clusterDF.members.apply(lambda x: len(x))
+            clusterDF['horizontal'] = clusterDF.apply(lambda x: x['ms'] == x['v-h'] and x['v-h'] > 1, axis=1)
+
+
+
+            logging.info('Spread horizontal %d vertical %d',len(clusterDF[clusterDF['horizontal'] == True]),  len(clusterDF[clusterDF['vertical'] == True]))
+            logging.info('Spread patch horizontal %d vertical %d',clusterDF[clusterDF['horizontal'] == True]['v-h'].sum(),  clusterDF[clusterDF['vertical'] == True]['v-h'].sum())
+            logging.info('Spread hunks horizontal %d vertical %d',clusterDF[clusterDF['horizontal'] == True]['ms'].sum(),  clusterDF[clusterDF['vertical'] == True]['ms'].sum())
+
+        singlePatternAll= clusterDF[clusterDF.members.apply(lambda x:haveAll(x))]
+        singlePatterns = list(set(singlePatternAll.cid.values.tolist()))
+
+        # if type == 'actions':
+        #     print('\n'.join(singlePatterns))
+        hunks = set(itertools.chain.from_iterable(singlePatternAll.members.values.tolist()))
+        logging.info("Match every member Fix\n %s # bugs: %d , # hunks %d ,#patterns %d", type,len(set([re.split('.txt_[0-9]+', i)[0] for i in hunks])),
+                     len(hunks),
+                     len(singlePatternAll))
+        # nonEmpty = clusterDF[clusterDF.members.str.len() > 1]
+        allhunks =set(itertools.chain.from_iterable(clusterDF.members.values.tolist()))
+        byCombi = combinationOfPatterns(allhunks)
+
+
+        multiple =clusterDF[clusterDF.members.apply(lambda x:test(x))]
+        # print(len(multiple))
+        # print(set(multiple.cid.values.tolist()).difference(singlePatterns))
+
+        allHunks
+        logging.info("%d patterns can match %d bugs, %d hunks",len(multiple),len(set([re.split('.txt_[0-9]+', i)[0] for i in allHunks])), len(allHunks))
+        matches = pd.DataFrame(statsS, columns=['cluster', 'memberCount'])
+        matches.sort_values(by='memberCount', ascending=False, inplace=True)
+        matches
+        if isFixminer:
+            matches.to_csv(join(DATA_PATH, "stats" + type + ".csv"), index=False)
+            if type == 'actions':
+                clusterDF['ms'] = clusterDF.members.str.len()
+                clusterDF.sort_values(by='ms', ascending=False, inplace=True)
+                top50 = clusterDF.head(50)
+                top50['member'] = top50.members.apply(lambda x: x[0])
+                top50['cid'] = top50.cid.apply(lambda x: x[0])
+                top50['path'] = top50.apply(lambda x:x['cid'].replace('-','/')+'/'+x['member'],axis=1)
+                def readFile(x):
+                    with open(join(DATA_PATH,'actions',x), 'r', encoding='utf-8') as writeFile:
+                        lines = writeFile.read()
+                        return lines
+                        # if lines.startswith('UPD'):
+                        #     return lines
+                        # else:
+                        #     return ''
+                    # return lines
+
+                top50['pattern'] = top50.path.apply(lambda x:readFile(x))
+                top50[['cid','pattern']].to_csv('actionsPattern2verify.csv',index=False,header=None)
+
+
+
+
+
+
+
+
+        else:
+            matches.to_csv(join(DATA_PATH, "statsDefects4J" + type + ".csv"), index=False)
+        print(type, matches.memberCount.sum())
+        print(type, matches.memberCount.sum())
+        yList.append(matches.memberCount.values.tolist())
+        colNames.append(type)
+    # if isFixminer:
+    #     from common.commons import plotBox
+    #     plotBox(yList,colNames,'dist_clusterMembers.pdf',False,False)
+    # for i in range(2, 21):
+    #     print('%d %d %d' % (matches[matches.memberCount >= i].memberCount.sum(), len(matches[matches.memberCount >= i]),i))
+
+    # save_zipped_pickle(join(DATA_PATH,'statsShapes'))
+
+idx = 0
+def stats(type,isFixminer=True):
+    clustersDF = pd.DataFrame(columns=['cid', 'type', 'members'])
+
+
+    shapesPath = join(DATA_PATH, type)
+    shapes = listdir(shapesPath)
+    shapes = [f for f in shapes if isdir(join(shapesPath, f))]
+    shape = size = cluster = action = token = None
+
+    def statsCore(cs):
+        global idx
+        if isFixminer:
+            cs = [i for i in cs if not (
+                    i.startswith('commons-math') or i.startswith('commons-lang') or i.startswith(
+                'closure-compiler') or i.startswith('joda-time') or i.startswith('mockito') or i.startswith(
+                'jfreechart'))]
+        else:
+            cs = [i for i in cs if (
+                    i.startswith('commons-math') or i.startswith('commons-lang') or i.startswith(
+                'closure-compiler') or i.startswith('joda-time') or i.startswith('mockito') or i.startswith(
+                'jfreechart'))]
+        if size == '1':
+            return
+        # print('Cluster %s : member size %s' % (shape+"-"+size +"-"+cluster, len(cs)))
+        clusterSize = len(cs)
+        if token is None:
+            if action is None:
+                # clusterSize = len(cs)
+                # if clusterSize > 1:
+                #     clusterSize = len(set([re.split('.txt_[0-9]+', i)[0] for i in cs]))
+                t = shape + "-" + size + "-" + cluster, clusterSize
+                clustersDF.loc[idx] = [t, type, cs]
+                idx = idx + 1
+            else:
+                # clusterSize = len(cs)
+                # if clusterSize > 1:
+                #     clusterSize = len(set([re.split('.txt_[0-9]+', i)[0] for i in cs]))
+                t = shape + "-" + size + "-" + cluster + "-" + action, clusterSize
+                clustersDF.loc[idx] = [t, type, cs]
+                idx = idx + 1
+        else:
+            # clusterSize = len(cs)
+            # if clusterSize > 1:
+            #     clusterSize = len(set([re.split('.txt_[0-9]+', i)[0] for i in cs]))
+            t = shape + "-" + size + "-" + cluster + "-" + action + "-" + token, clusterSize
+            clustersDF.loc[idx] = [t, type, cs]
+            idx = idx + 1
+        # t = shape + "-" + size + "-" + cluster, len(cs)
+        # if len(cs)>0:
+        #     with open(join(shapesPath,t[0].replace('-','/'),cs[0]),'r') as pattern:
+        #         line = pattern.read()
+        #         if line.startswith('MOV') or line.startswith('DEL'):
+        #             t = t[0],0
+        if isFixminer:
+            if clusterSize > 1:
+                statsS.append(t)
+        else:
+            if clusterSize > 0:
+                statsS.append(t)
+
+    statsS = []
+    for shape in shapes:
+        if shape.startswith('.'):
+            continue
+        sizes = listdir(join(shapesPath, shape))
+
+        for size in sizes:
+            if size.startswith('.'):
+                continue
+            clusters = listdir(join(shapesPath, shape, size))
+            for cluster in clusters:
+                if cluster.startswith('.'):
+                    continue
+                cs = listdir(join(shapesPath, shape, size, cluster))
+
+                if shapesPath.endswith('shapes'):
+                    cs = listdir(join(shapesPath, shape, size, cluster))
+                    statsCore(cs)
+                else:
+                    # level3
+                    for action in cs:
+                        if action.startswith('.'):
+                            continue
+                        tokens = listdir(join(shapesPath, shape, size, cluster, action))
+                        if shapesPath.endswith('actions'):
+                            statsCore(tokens)
+                        else:
+                            for token in tokens:
+                                if token.startswith('.'):
+                                    continue
+                                cs = listdir(join(shapesPath, shape, size, cluster, action, token))
+                                statsCore(cs)
+    return statsS,clustersDF
+
+
+def defects4jStats(isFixminer=False):
+    if (isfile(join(DATA_PATH, 'defects4j-mapping.pickle'))):
+        matches = load_zipped_pickle(join(DATA_PATH, 'defects4j-mapping.pickle'))
+    else:
+        # defects4j mapping
+        mapping = pd.read_csv('mapping.csv', header=None, index_col=None, sep=' ')
+        mapping.rename(columns={0: 'repo', 1: "commit", 2: 'defects4jID'}, inplace=True)
+        dbDir = join(DATA_PATH, 'redis')
+
+        portInner = '6399'
+        startDB(dbDir, portInner, PROJECT_TYPE )
+
+        import redis
+
+        redis_db = redis.StrictRedis(host="localhost", port=portInner, db=0)
+        keys = redis_db.scan(0, match='*', count='1000000')
+
+        matches = pd.DataFrame(keys[1], columns=['pairs_key'])
+
+        # matches = load_zipped_pickle(join(DATA_PATH,'singleHunks'))
+        matches['pairs_key'] = matches['pairs_key'].apply(lambda x: x.decode())
+        matches['root'] = matches['pairs_key'].apply(lambda x: x.split('/')[0])
+        matches['size'] = matches['pairs_key'].apply(lambda x: x.split('/')[1])
+        matches['file'] = matches['pairs_key'].apply(lambda x: x.split('/')[2])
+        matches['repo'] = matches['file'].apply(lambda x: x.split('_')[0])
+        matches['commit'] = matches['file'].apply(lambda x: x.split('_')[2])
+        matches['hunk'] = matches['pairs_key'].apply(lambda x: x.split('/')[2].split('_')[-1])
+        matches['fileName'] = matches['pairs_key'].apply(lambda x: '_'.join(x.split('/')[2].split('_')[:-1]))
+
+        # save_zipped_pickle(matches, join(DATA_PATH, 'matches.pickle'))
+        matches = matches[matches.repo.apply(lambda i: (
+                    i.startswith('commons-math') or i.startswith('commons-lang') or i.startswith(
+                'closure-compiler') or i.startswith('joda-time') or i.startswith('mockito') or i.startswith('jfreechart')))]
+        # matches = matches[matches.repo.apply(lambda i:  (i.endswith('.git')))]
+        matches['defects4jID'] = matches.apply(lambda x: mapping.query(
+            "commit.str.startswith('{0}') and repo== '{1}'".format(x['commit'], x['repo'])).defects4jID.tolist(), axis=1)
+        save_zipped_pickle(matches, join(DATA_PATH, 'defects4j-mapping.pickle'))
+
+    if not isfile(join(DATA_PATH, 'defects4jcluster.pickle')):
+        # matches = load_zipped_pickle(join(DATA_PATH,'defects4j-mapping.pickle'))
+        # clustersDF = pd.DataFrame(columns=['cid', 'type', 'members'])
+        # idx = 0
+        clustersL = []
+        for type in ['tokens', 'actions', 'shapes']:
+            statsS, clustert = stats(type, isFixminer)
+            clustert =clustert[clustert.members.apply(lambda x:len(x)>0)]
+            clustersL.append(clustert)
+        clustersDF = pd.concat(clustersL)
+
+        # clustersDF
+        # number of instances
+        # clustersDF[clustersDF.type == 'tokens'].members.str.len().sum()
+        # cluster len
+        # len(clustersDF[clustersDF.type == 'shapes'])
+        matches['defects4jID'] = matches['defects4jID'].apply(lambda x: x[0])
+
+
+        def getDefects4JID(x):
+            # filenames = list(set([re.split('.txt_[0-9]+', i)[0] for i in x]))
+            # bids2Compare = [matches[matches.file.str.startswith(fn)].defects4jID.unique()[0] for fn in filenames]
+            keys = []
+            for fn in x:
+                selected = matches[matches.file == fn]
+                if len(selected) != 1:
+                    print('erro')
+                else:
+                    key = selected.iloc[0]['repo'] + '-' + str(selected.iloc[0]['defects4jID'])
+                    keys.append(key)
+
+            return list(set(keys))
+
+
+        clustersDF['defects4j'] = clustersDF.members.apply(lambda x: getDefects4JID(x))
+        p
+        save_zipped_pickle(clustersDF, join(DATA_PATH, 'defects4jcluster.pickle'))
+    else:
+        clustersDF = load_zipped_pickle(join(DATA_PATH, 'defects4jcluster.pickle'))
+    clustersDF
+    clustersDF['ms'] = clustersDF.members.str.len()
+
+    for t in ['shapes', 'actions', 'tokens']:
+        ds = clustersDF[clustersDF.type == t]
+        ds.sort_values(by='ms', ascending=False, inplace=True)
+        # ds = ds[ds.ms > 1]
+
+        ds[['cid', 'ms', 'defects4j']].to_csv(join(DATA_PATH, 'dissectionDefects4j' + t + '.csv'), index=None, header=None)
+
+        print(t, ds.ms.sum(), len(ds))
+        hunks = list(itertools.chain.from_iterable(ds.members.values.tolist()))
+        bugs = set(list(itertools.chain.from_iterable(ds.defects4j.values.tolist())))
+        print(len(hunks), len(bugs))
+
+        keys = [tuple(i.rsplit('-', 1)) for i in bugs]
+
+        test = pd.read_json(join(DATA_PATH, 'defects4j-bugs.json'))
+        test['newKey'] = test.apply(lambda x: tuple([x['program'], str(x['bugId'])]), axis=1)
+        selectBugs = test[test.newKey.isin(keys)]
+        patterns = selectBugs[['program', 'bugId', 'repairPatterns']]
+        patterns['actions'] = selectBugs['repairActions']
+        patterns['oId'] = patterns.apply(lambda x: x['program'] + '-' + str(x['bugId']), axis=1)
+        patterns['oId'].apply(lambda x: ds[ds.defects4j.apply(lambda y: x in y)].cid.values.tolist())
+        patterns['myPatterns'] = patterns['oId'].apply(
+            lambda x: ds[ds.defects4j.apply(lambda y: x in y)].cid.values.tolist())
+        classification = pd.read_json(join(DATA_PATH, 'classification.json'))
+
+        classKeys = classification[~classification['Repair Patterns'].isna()]['Repair Patterns'].values.tolist()
+        classKeys = list(itertools.chain.from_iterable([list(i.keys()) for i in classKeys]))
+        classKeys.remove('notClassified')
+        patterns.repairPatterns = patterns.repairPatterns.apply(lambda x: [i for i in x if i in classKeys])
+        dissectionPattern = set(itertools.chain.from_iterable(patterns['repairPatterns'].values.tolist()))
+
+        dissection = ds
+        dissection['cid'] = dissection.cid.apply(lambda x: x[0])
+        dissection = dissection[['cid', 'defects4j']]
+        dissection['repairPatterns'] = dissection.defects4j.apply(lambda x:set([tuple(patterns[patterns.oId == i].iloc[0].repairPatterns) for i in x]))
+        notSingle = dissection[dissection.defects4j.str.len() != 1]
+
+        notSingle['repairPatterns'] = notSingle.repairPatterns.apply(lambda x:set.intersection(*[set(i) for i in x]))
+        notSingle = notSingle[notSingle.repairPatterns != set()]
+        notSingle.repairPatterns =notSingle.repairPatterns.apply(lambda i: {tuple(i)})
+        # myDataset = notSingle[notSingle.repairPatterns.str.len() == 1]
+        myDataset = dissection[dissection.defects4j.str.len() == 1]
+        myPatterns = pd.concat([notSingle, myDataset])
+
+        print(len(set.union(*myPatterns.repairPatterns.values.tolist())))
+
+        myPatterns.repairPatterns = myPatterns.repairPatterns.apply(lambda x: list(x.pop()))
+        allPAtterns =  myPatterns.repairPatterns.values.tolist()
+        [i.sort() for i in allPAtterns]
+
+        counts = Counter([tuple(i) for i in allPAtterns]).items()
+        granularity = {k: v for k, v in counts if v > 1 and k != tuple()}
+        print('consisteny %' ,sum([v for k, v in counts if k != tuple()]))
+        print('Granularity %', len(granularity))
+
+
+
+        myPatterns.to_csv(join(DATA_PATH, 'dissectionMyPatterns' + t + '.csv'), index=None, header=None)
+
+        # dissectionPatterns = [list(i) for i in set.union(*myPatterns.repairPatterns.values.tolist())]
+        # [i.sort() for i in dissectionPatterns]
+        # print(len(set.union(*[{tuple(i)} for i in dissectionPatterns])))
+        # logging.info('%s Unique label %d',t, len(set(list(itertools.chain.from_iterable(myDataset.repairPatterns.values.tolist())))))
+        # def cmp(a, b):
+        #     if a > b:
+        #         return 'Dissection'
+        #     elif a == b:
+        #         return 'Equal'
+        #     else:
+        #         return 'Fixminer'
+        #
+        #
+        # patterns['cmp'] = patterns.apply(lambda x: cmp(len(x['repairPatterns']), len(x['myPatterns'])), axis=1)
+        # patterns.to_csv(join(DATA_PATH, 'dissectionPatterns' + t + '.csv'), index=None, header=None)
+        # print('Pattern')
+        # print(patterns['cmp'].value_counts())
+        # patterns['cmpActions'] = patterns.apply(lambda x: cmp(len(x['actions']), len(x['myPatterns'])), axis=1)
+        # print('Action')
+        # print(patterns['cmpActions'].value_counts())
+        # myPatterns = set(itertools.chain.from_iterable(patterns['myPatterns'].values.tolist()))
+        # print(len(dissectionPattern), len(myPatterns))
+        # # classification = pd.read_json(join(DATA_PATH, 'classification.json'))
+        #
+        # patterns.repairPatterns = patterns.repairPatterns.apply(lambda x: tuple(x))
+        # logging.info('%s # Dissection Patterns %d, Unique %d, Empty %d',t, len(patterns['repairPatterns'].values.tolist()) -patterns['repairPatterns'].values.tolist().count(()),
+        # len(set(patterns['repairPatterns'].values.tolist())),patterns['repairPatterns'].values.tolist().count(()))
+        #
+        # myPatterns = patterns.myPatterns.str.len().values.tolist()
+        # repairPatterns = patterns.repairPatterns.str.len().values.tolist()
+        # actions = patterns.actions.str.len().values.tolist()
+        #
+        #
+        #
+        #
+        # plotScatter(myPatterns,repairPatterns,"# FixMiner Patterns",'# Dissection Patterns',11,t+'Fixminer-Patterns')
+        # plotScatter(myPatterns,actions,"# FixMiner Patterns",'# Dissection Patterns',21,t+'Fixminer-Actions')
+
+
+javaAst = ["AnonymousClassDeclaration", "ArrayAccess", "ArrayCreation", "ArrayInitializer", "ArrayType", "AssertStatement",
+       "Assignment", "Block", "BooleanLiteral", "BreakStatement", "CastExpression", "CatchClause", "CharacterLiteral",
+       "ClassInstanceCreation", "CompilationUnit", "ConditionalExpression", "ConstructorInvocation",
+       "ContinueStatement", "DoStatement", "EmptyStatement", "ExpressionStatement", "FieldAccess", "FieldDeclaration",
+       "ForStatement", "IfStatement", "ImportDeclaration", "InfixExpression", "Initializer", "Javadoc",
+       "LabeledStatement", "MethodDeclaration", "MethodInvocation", "NullLiteral", "NumberLiteral",
+       "PackageDeclaration", "ParenthesizedExpression", "PostfixExpression", "PrefixExpression", "PrimitiveType",
+       "QualifiedName", "ReturnStatement", "SimpleName", "SimpleType", "SingleVariableDeclaration", "StringLiteral",
+       "SuperConstructorInvocation", "SuperFieldAccess", "SuperMethodInvocation", "SwitchCase", "SwitchStatement",
+       "SynchronizedStatement", "ThisExpression", "ThrowStatement", "TryStatement", "TypeDeclaration",
+       "TypeDeclarationStatement", "TypeLiteral", "VariableDeclarationExpression", "VariableDeclarationFragment",
+       "VariableDeclarationStatement", "WhileStatement", "InstanceofExpression", "LineComment", "BlockComment",
+       "TagElement", "TextElement", "MemberRef", "MethodRef", "MethodRefParameter", "EnhancedForStatement",
+       "EnumDeclaration", "EnumConstantDeclaration", "TypeParameter", "ParameterizedType", "QualifiedType",
+       "WildcardType", "NormalAnnotation", "MarkerAnnotation", "SingleMemberAnnotation", "MemberValuePair",
+       "AnnotationTypeDeclaration", "AnnotationTypeMemberDeclaration", "Modifier", "UnionType", "Dimension",
+       "LambdaExpression", "IntersectionType", "NameQualifiedType", "CreationReference", "ExpressionMethodReference",
+       "SuperMethodReference", "TypeMethodReference", "MethodName", "Operator", "New", "Instanceof"]
+
+cAst = ["unit","comment","literal","operator","modifier","name","type","condition","block","index","decltype","typename","atomic","assert","generic_selection","selector","association_list","association","expr_stmt","expr","decl_stmt","decl","init","range","break","continue","goto","label","typedef","asm","macro","enum","enum_decl","if","ternary","then","else","elseif","while","typeof","do","switch","case","default","for","foreach","control","incr","function","function_decl","lambda","specifier","return","call","sizeof","parameter_list","parameter","krparameter_list","krparameter","argument_list","argument","capture","struct","struct_decl","union","union_decl","class","class_decl","public","private","protected","signals","forever","emit","member_init_list","constructor","constructor_decl","destructor","destructor_decl","super","friend","extern","namespace","using","try","catch","finally","throw","throws","noexcept","template","directive","file","number","include","define","undef","line","ifdef","ifndef","elif","endif","pragma","error","warning","value","empty","region","endregion","import","marker","parse","mode","lock","fixed","checked","unchecked","unsafe","using_stmt","delegate","event","constraint","extends","implements","package","synchronized","interface","interface_decl","annotation_defn","static","attribute","target","linq","from","select","where","let","orderby","group","join","in","on","equals","by","into","escape","annotation","alignas","alignof","typeid","ref_qualifier","receiver","message","protocol_list","category","protocol","required","optional","property","attribute_list","synthesize","dynamic","encode","autoreleasepool","compatibility_alias","protocol_decl","cast","position","clause","empty_stmt"]
+
+
+def exportAbstractPatterns():
+    clusterStats,df = stats('shapes')
+    port = 6399
+    import redis
+    redis_db = redis.StrictRedis(host="localhost", port=port, db=0)
+    isJava = False
+    if 'java' == PROJECT_TYPE:
+        isJava = True
+    for id, members in df[['cid','members']].values.tolist():
+
+
+        dKey = '/'.join(id[0].split('-')[:-1]) + "/" + members[0]
+        lines = redis_db.hget("dump",dKey )
+        cid = id[0].replace("-",'#')
+        abstractPattern(cid,lines.decode(),isJava,members)
+
+def abstractPattern(cid,lines,isJava,cMembers):
+
+    if isJava:
+        ast = javaAst
+    else:
+        ast = cAst
+    movPattern = 'MOV (' + '|'.join(ast) + ')@@(.*)@TO@ (' + '|'.join(ast) + ')@@(.*)@AT@'
+    delPattern = 'DEL (' + '|'.join(ast) + ')@@(.*)@AT@'
+    insPattern = 'INS (' + '|'.join(ast) + ')@@(.*)@TO@ (' + '|'.join(ast) + ')@@(.*)@AT@'
+    updPattern = 'UPD (' + '|'.join(ast) + ')@@(.*)@TO@(.*)@AT@'
+
+    level = 'actions'
+
+    lines = re.split("@LENGTH@ \d+", lines)
+    tokens = []
+    for line in lines:
+        # levelPatch  = len(re.findall('\w*---', line))
+        match = re.search(r"^\w*---+", line,re.M)
+
+
+        if match is not None:
+            not_matched, matched = line[:match.start()], match.group()
+            levelPatch  = int(len(matched) / 3)
+        else:
+            levelPatch = 0
+        line = line.strip().strip('-')
+        type = ''
+        if line is '':
+            continue
+        t = []
+        searchPattern = ''
+        if line.startswith('INS'):
+            if level =='actions':
+                t= [1,3]
+            else:
+                t = [1]
+            searchPattern = insPattern
+            type =' INS '
+        elif line.startswith('UPD'):
+            t = [1]
+            searchPattern = updPattern
+            type = ' UPD '
+        elif line.startswith('DEL'):
+            t = [1]
+            searchPattern = delPattern
+            type = ' DEL '
+        elif line.startswith('MOV'):
+            if level == 'actions':
+                t = [1, 3]
+            else:
+                t = [1]
+            searchPattern = movPattern
+            type = ' MOV '
+    # from common.preprocessing import preprocessingForSimi
+        m = re.search(searchPattern, line, re.DOTALL)
+        if t is None:
+            print()
+        if m:
+            for k in t:
+                prefix = '---' * levelPatch
+                if prefix != '':
+                    prefix = '\n'+prefix
+                token = m.group(k)
+                if level =='actions':
+                    if k ==3:
+                        prefix = 'TO '
+                    else:
+                        prefix = prefix + type
+
+                tokens.append(prefix+token)
+
+    tokens
+    inferedFrom = "\n\n // Infered from ({})".format(', '.join(cMembers))
+    tokens.append(inferedFrom)
+    clusterSavePath = join(DATA_PATH,'patterns')
+    os.makedirs(clusterSavePath, exist_ok=True)
+    with open(join(clusterSavePath, cid), 'w', encoding='utf-8') as writeFile:
+    #     # if levelPatch == 0:
+        writeFile.write(' '.join(tokens))
+        # else:
+        #     writeFile.write('\n'.join(tokens))

python/test_patched_file.py

@@ -0,0 +1,247 @@
+import bugzoo
+from bugzoo import server, Container
+from common.commons import *
+import signal
+
+# from common.commons import shellGitCheckout
+DATA_PATH = os.environ["DATA_PATH"]
+introClassFile = join(DATA_PATH,'introClassData.txt')
+
+def testCore(t):
+        bugName, port = t
+        container = None
+        # with bugzoo.server.ephemeral(port=port, verbose=False,bugzooPath="/Users/anil.koyuncu/anaconda3/envs/python36/bin/bugzood", timeout_connection=3000) as client:
+        cmd = 'bash {} {}'.format(join(DATA_PATH,'startBugzoo.sh'),port)
+
+        with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) as process:
+
+            # o,e = shellGitCheckout(cmd)
+            url = "http://127.0.0.1:{}".format(port)
+            timeout_connection = 3000
+            client = Client(url, timeout_connection=timeout_connection)
+            # with bugzoo.server.ephemeral(port=port, verbose=False, timeout_connection=3000) as client:
+            try:
+                # port = 6060
+                # url = "http://127.0.0.1:{}".format(port)
+                # client = Client(url, timeout_connection=300)
+                # bugName = bugList[i]
+                # if bugName != 'introclass:digits:070455:000':
+                #     continue
+                fix = 'failure'
+                output = ''
+                # print("bugName: {}".format(bugName), end=' ')
+                output += 'bugName:' + bugName + ', '
+
+                bug = client.bugs[bugName]
+                if not client.bugs.is_installed(bug):
+                    output += ' building'
+                    client.bugs.build(bug)
+
+                    # client.bugs.download(bug)
+                    # print("the image is not installed :'(")
+
+                # print("creating container...")
+                container = client.containers.provision(bug)
+                # print("container is ready")
+
+                # print("First_test:", end=' ')
+                output += 'First_test:'
+                pre_test_outcomes = {}
+                pre_failure_cases, pre_failure, total, pre_test_outcomes = test_all(bug, container, client)
+                # print("@fail:{}@total:{}".format(pre_failure, total), end=' ')
+                # print("@pre_failure_cases:{}".format(pre_failure_cases), end=' ')
+                if pre_failure == 0:
+                    logging.error(bugName + ' no failed test initially')
+                    return ''
+                output += '@fail:' + str(pre_failure) + '@total:' + str(total) + ', '
+
+                # print("patching...")
+                path = join(DATA_PATH,'introclass',bugName)
+                patch_path = join(path ,'patched')
+                # avaliable_patch = os.path.abspath('data') + '/introclass2/' + bugName + '/' + 'patches'
+                patch_names = os.listdir(patch_path)
+
+                times = 0
+                for patch_name in patch_names:
+                    # if patch_name not in os.listdir(avaliable_patch):
+                    #     continue
+                    patch = join(patch_path,patch_name)
+
+                    patch_result = patched_application(path, bug.name, patch, client, container)
+                    if patch_result == -1 or patch_result.code != 0:
+                        # print("@{}@".format('False'), end='')
+                        # print("{}".format('F'), end=' ')
+                        output += '@False@F '
+                        continue
+                    # print("@{}@".format('True'), end='')
+                    output += '@True@'
+
+                    # print("Second_test:", end=' ')
+                    post_test_outcomes = {}
+                    post_failure_cases, post_failure, total, post_test_outcomes = test_all(bug, container, client)
+                    # print("{}".format(post_failure), end=' ')
+                    output += str(post_failure) + ' '
+                    if post_failure == 0:
+                        times += 1
+                        fix = 'success'
+                        # print("fix {} by {}".format(bugName, patch_name))
+                        output += 'fix {} by {} '.format(bugName, patch_name)
+                    # print("@fail:{}@total:{}".format(post_failure, total),end=' ')
+                    # print("@post_failure_cases:{}".format(post_failure_cases))
+
+
+                # cmd = 'docker rm -fv {}'.format(container.id)
+                # out, e = shellGitCheckout(cmd)
+
+                output += 'times:{}, '.format(times) + fix
+                print(output)
+                return output
+            except Exception as e:
+                logging.error(e)
+            finally:
+                # ''
+                cmd = 'docker stop {}'.format(container.id)
+                out, e = shellGitCheckout(cmd)
+                client.shutdown()
+                os.killpg(process.pid, signal.SIGTERM)
+                # docker stop $(docker ps -q)
+
+
+
+def patch_validate():
+
+
+    logger = logging.getLogger()
+
+    for k,v in logger.manager.loggerDict.items():
+        if(k.startswith('bugzoo')):
+            if isinstance(v,logging.Logger):
+                v.setLevel(logging.ERROR)
+
+    # url = "http://127.0.0.1:6060"
+    # client = bugzoo.Client(url)
+    # bug = client.bugs['introclass:checksum:08c7ea:006']
+    bugList = []
+
+    # cmd = 'bash ' + join(DATA_PATH,'startBugzoo.sh')
+    # cmd = "/Users/anil.koyuncu/anaconda3/envs/python36/bin/bugzood --debug -p " + str(port)
+    # output, errors = shellGitCheckout(cmd)
+    # with bugzoo.server.ephemeral(port=8082, verbose=True, timeout_connection=3000) as client:
+    port = 6000
+    bugs2test= listdir(join(DATA_PATH, "introclass"))
+    for b in bugs2test:
+        t = b, port
+        bugList.append(t)
+        if port == 6300:
+            port = 6000
+        port += 1
+    # with open(introClassFile, 'r') as file:
+    #     for line in file.readlines():
+    #
+    #         t =line.strip(),port
+    #         bugList.append(t)
+    #         if port == 6300:
+    #             port = 6000
+    #         port +=1
+
+        # for i in range(0,len(bugList)):
+    # t = 'introclass:syllables:93f87b:005',6000
+    # t = 'introclass:syllables:99cbb4:000',6000
+    # testCore(t)
+    results = parallelRunMerge(testCore, bugList,max_workers=10)
+    # print('\n'.join(results))
+    with open(join(DATA_PATH, 'introTestResults'), 'w',
+              encoding='utf-8') as writeFile:
+        # if levelPatch == 0:
+        writeFile.write('\n'.join(results))
+        # for i in bugList:
+        #     testCore(i)
+        # pass
+
+
+from bugzoo import Patch, Client
+
+def test_all(bug, container,  client):
+    test_outcomes = {}  # type: Dict[TestCase, TestOutcome]
+    failure_cases = []
+    failure = 0
+    total = len(bug.tests._tests)
+    for test in bug.tests:
+        test_outcomes[test] = client.containers.test(container, test)
+        # if test.expected_outcome != test_outcomes[test].passed:
+        if test_outcomes[test].passed != True:
+            failure += 1
+            failure_cases.append(test.command)
+            break
+    return failure_cases, failure, total, test_outcomes
+
+
+def patched_application(path, bugName, patched, client: Client, container: Container):
+    buggroup = bugName.split(":")[1]
+    # path = join(BUGDIR,bug)
+    program = path + '/' + buggroup + '.c'
+    fixedFile = patched.split('/')[-1]
+
+    cmd = 'docker cp ' + patched + ' ' + container.id + ':/experiment/'
+    output, e = shellGitCheckout(cmd)
+
+    cmd = 'sudo chown $(whoami):$(whoami) "{}"'
+    cmd = cmd.format(fixedFile)
+    output = client.containers.exec(container=container, command=cmd, context='/experiment/')
+    # cmd = 'rm .genprog_test_cache.json & mv {} {}.c & gcc -o {} {}'.format(fixedFile,fixedFile, buggroup, fixedFile+'.c')
+    cmd = 'rm {} & rm .genprog_test_cache.json'.format(buggroup)
+    output = client.containers.exec(container=container, command=cmd, context='/experiment/')
+
+    cmd = 'setarch `uname -m` -R gcc -o {} {}'.format(buggroup, fixedFile)
+    output = client.containers.exec(container=container, command=cmd, context='/experiment/')
+    return output
+
+def checkCorrect():
+    regex = r"fix (.*) by (.*)\.c"
+    if not os.path.exists(join(DATA_PATH, 'introclass_eval')):
+        os.makedirs(join(DATA_PATH, 'introclass_eval'))
+
+    with open(join(DATA_PATH, 'introTestResults'), 'r',
+              encoding='utf-8') as readFile:
+        res = readFile.readlines()
+        results = []
+        for line in res:
+            if line.strip().endswith('success'):
+                line = '\n'.join(line.split('@'))
+                matches = re.finditer(regex, line, re.MULTILINE)
+
+                for matchNum, match in enumerate(matches, start=1):
+
+
+                    bug, patch = match.groups()
+                    print(bug,patch)
+                    t = bug,patch
+                    results.append(t)
+
+                    # for groupNum in range(0, len(match.groups())):
+                    #     groupNum = groupNum + 1
+                    #
+                    #     print("Group {groupNum} found at {start}-{end}: {group}".format(groupNum=groupNum,
+                    #                                                                     start=match.start(groupNum),
+                    #                                                                     end=match.end(groupNum),
+                    #                                                                     group=match.group(groupNum)))
+                    #     line
+        for t in results:
+            bug, patch = t
+
+            if not os.path.exists(join(DATA_PATH, 'introclass_eval',bug)):
+                os.makedirs(join(DATA_PATH, 'introclass_eval',bug))
+
+            buggyFile = join(DATA_PATH,'introclass',bug,patch.split('.')[0]+'.c')
+            spinferFix = join(DATA_PATH,'introclass_patched',bug,'patches',patch + '.c')
+            spinferPatch = join(DATA_PATH,'introclass_eval',bug,patch +'.patch')
+            oracle = join(DATA_PATH,'introclass',bug,'oracle.c.patch')
+            oracleEval = join(DATA_PATH,'introclass_eval',bug,'oracle.patch')
+            cmd =  'diff -u ' +buggyFile + ' ' + spinferFix + ' > ' + spinferPatch
+            o,e= shellGitCheckout(cmd)
+            shutil.copy(oracle,oracleEval)
+
+
+if __name__ == "__main__":
+    patch_validate()
+

python/tokens.py

@@ -0,0 +1,138 @@
+from common.commons import *
+
+DATA_PATH = os.environ["DATA_PATH"]
+ast = ["AnonymousClassDeclaration", "ArrayAccess", "ArrayCreation", "ArrayInitializer", "ArrayType", "AssertStatement",
+       "Assignment", "Block", "BooleanLiteral", "BreakStatement", "CastExpression", "CatchClause", "CharacterLiteral",
+       "ClassInstanceCreation", "CompilationUnit", "ConditionalExpression", "ConstructorInvocation",
+       "ContinueStatement", "DoStatement", "EmptyStatement", "ExpressionStatement", "FieldAccess", "FieldDeclaration",
+       "ForStatement", "IfStatement", "ImportDeclaration", "InfixExpression", "Initializer", "Javadoc",
+       "LabeledStatement", "MethodDeclaration", "MethodInvocation", "NullLiteral", "NumberLiteral",
+       "PackageDeclaration", "ParenthesizedExpression", "PostfixExpression", "PrefixExpression", "PrimitiveType",
+       "QualifiedName", "ReturnStatement", "SimpleName", "SimpleType", "SingleVariableDeclaration", "StringLiteral",
+       "SuperConstructorInvocation", "SuperFieldAccess", "SuperMethodInvocation", "SwitchCase", "SwitchStatement",
+       "SynchronizedStatement", "ThisExpression", "ThrowStatement", "TryStatement", "TypeDeclaration",
+       "TypeDeclarationStatement", "TypeLiteral", "VariableDeclarationExpression", "VariableDeclarationFragment",
+       "VariableDeclarationStatement", "WhileStatement", "InstanceofExpression", "LineComment", "BlockComment",
+       "TagElement", "TextElement", "MemberRef", "MethodRef", "MethodRefParameter", "EnhancedForStatement",
+       "EnumDeclaration", "EnumConstantDeclaration", "TypeParameter", "ParameterizedType", "QualifiedType",
+       "WildcardType", "NormalAnnotation", "MarkerAnnotation", "SingleMemberAnnotation", "MemberValuePair",
+       "AnnotationTypeDeclaration", "AnnotationTypeMemberDeclaration", "Modifier", "UnionType", "Dimension",
+       "LambdaExpression", "IntersectionType", "NameQualifiedType", "CreationReference", "ExpressionMethodReference",
+       "SuperMethodReference", "TypeMethodReference", "MethodName", "Operator", "New", "Instanceof"]
+
+movPattern = 'MOV (' + '|'.join(ast) + ')@@(.*)@TO@ (' + '|'.join(ast) + ')@@(.*)@AT@'
+delPattern = 'DEL (' + '|'.join(ast) + ')@@(.*)@AT@'
+insPattern = 'INS (' + '|'.join(ast) + ')@@(.*)@TO@ (' + '|'.join(ast) + ')@@(.*)@AT@'
+updPattern = 'UPD (' + '|'.join(ast) + ')@@(.*)@TO@(.*)@AT@'
+
+import redis
+port = '6380'
+redis_db = redis.StrictRedis(host="localhost", port=port, db=0)
+redis_db1 = redis.StrictRedis(host="localhost", port=port, db=1)
+redis_db2 = redis.StrictRedis(host="localhost", port=port, db=2)
+
+redis_out  = redis.StrictRedis(host="localhost", port=6399, db=0)
+
+def getTokens(prefix, i):
+    dist2load = redis_db1.get(prefix + "-" + i);
+
+    with open(join(DATA_PATH, 'actions', prefix.replace('-', '/'), dist2load.decode()), 'r') as rFile:
+        lines = rFile.read()
+
+    lines = re.split("@LENGTH@ \d+", lines)
+    tokens = []
+    for line in lines:
+        line = line.strip().strip('-')
+        if line is '':
+            continue
+        t = []
+        searchPattern = ''
+        if line.startswith('INS'):
+            t = [2]
+            searchPattern = insPattern
+        elif line.startswith('UPD'):
+            t = [2, 3]
+            searchPattern = updPattern
+        elif line.startswith('DEL'):
+            t = [2]
+            searchPattern = delPattern
+        elif line.startswith('MOV'):
+            t = [2]
+            searchPattern = movPattern
+
+        # MOV TryStatement@@try:[] @TO@ MethodDeclaration@@public, T, T, MethodName:lookupByNameAndType, String name, Class<T> type,  @AT@ 2164 @LENGTH@ 646
+        # DEL VariableDeclarationStatement@@Path hfilePath=HFileLink.getHFileFromBackReference(getConf(),filePath); @AT@ 2474 @LENGTH@ 74
+        # INS ThrowStatement@@MethodInvocation:convertJedisAccessException(ex) @TO@ CatchClause@@Exception ex @AT@ 12194 @LENGTH@ 38
+        # UPD MethodInvocation@@getVectorExpression(elseDesc,mode) @TO@ getVectorExpression(elseDesc,VectorExpressionDescriptor.Mode.PROJECTION) @AT@ 136925 @LENGTH@ 35
+        # from common.preprocessing import preprocessingCodeElementsList
+
+        # lines = re.sub('@AT@\s*[0-9]+\s*', ' ', lines)
+        # lines = re.sub('@LENGTH@\s*[0-9]+\s*', ' ', lines)
+        # lines = re.sub('@TO@', ' ', lines)
+        # lines = re.sub('@@', ' ', lines)
+        # lines = re.sub('INS|UPD|MOV|DEL', ' ', lines)
+        # lines = re.sub('MethodInvocation:', ' ', lines)
+        # lines = re.sub('Name:', ' ', lines)
+        # lines = re.sub('|'.join(ast),' ',lines)
+        from common.preprocessing import preprocessingForSimi
+        m = re.search(searchPattern, line, re.DOTALL)
+        if t is None:
+            print()
+        if m:
+            for k in t:
+                token = m.group(k)
+                token = re.sub('MethodInvocation:|Name:|MethodName:|SimpleName:|InfixExpression:', ' ',
+                               token)
+                # token = re.sub(' Name:', ' ', token)
+                # token = re.sub(' MethodName:', ' ', token)
+                # token = re.sub(' SimpleName:', ' ', token)
+                tokens.append(token)
+
+
+        else:
+            return None
+
+    tokens = preprocessingForSimi(tokens)
+    return tokens
+    # for key in keys:
+
+
+def simiCore(key):
+    split = key.split('_')
+    prefix = split[0]
+    i = split[1]
+    j = split[2]
+
+
+    # inner = innerPool.getResource();
+
+    # preCorpusBug = preprocessingCodeElementsList(lines)
+    # return preCorpusBug
+
+    tokensi = getTokens(prefix, i)
+    tokensj = getTokens(prefix, j)
+
+    tokensi
+    import textdistance
+    # simi = textdistance.jaccard(tokensi,tokensj)
+    # simi2 = textdistance.sorensen_dice(' '.join(tokensi), ' '.join(tokensj))
+    simi2 = textdistance.sorensen_dice(list(unique_everseen(tokensi)), list(unique_everseen(tokensj)))
+    # simi
+    #
+    # from common.preprocessing import calculateTfIdfNLList
+    #
+    # if len(tokensj) == 0:
+    #     print()
+    # if tokensi[0] != [] or tokensj[0] != []:
+    #     v = calculateTfIdfNLList([tokensi])
+    #     sourceDTM = v.transform([tokensi])
+    #     bugDTM = v.transform([tokensj])
+    #     from sklearn.metrics.pairwise import cosine_similarity
+    #
+    #     res = cosine_similarity(bugDTM, sourceDTM)
+    #     simiScore =res[0][0]
+    if simi2 >= 0.8:
+        print(key,simi2)
+
+        redis_db2.set(key, simi2)
+    redis_db.delete(key)

python/validate_manybugs.py

@@ -0,0 +1,395 @@
+import bugzoo
+from bugzoo import server, Container
+import csv
+import os
+from common.commons import *
+import signal
+DATA_PATH = os.environ["DATA_PATH"]
+
+BUGDIR = join(DATA_PATH,'manybugs')
+PATCHES_DIR = join(DATA_PATH,'manybugs_sos')
+
+def patch_validate(t):
+    bugName, port = t
+    # with bugzoo.server.ephemeral(port=8080,verbose=True,timeout_connection=3000) as client:
+        # url = "http://127.0.0.1:6060"
+        # client = bugzoo.Client(url)
+        # bug = client.bugs['introclass:checksum:08c7ea:006']
+    # client,process = getClient(port)
+    container = None
+    cmd = 'bash {} {}'.format(join(DATA_PATH, 'startBugzoo.sh'), port)
+
+    with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) as process:
+        # o,e = shellGitCheckout(cmd)
+        url = "http://127.0.0.1:{}".format(port)
+        timeout_connection = 3000
+        client = Client(url, timeout_connection=timeout_connection)
+
+        try:
+            # bugName = 'manybugs:python:69223-69224'
+            if bugName == 'manybugs:php:2011-03-25-8138f7de40-3acdca4703' or bugName == 'manybugs:php:2011-10-31-c4eb5f2387-2e5d5e5ac6':
+                return
+
+            #get tests-list
+            group = bugName.split(':')[1]
+            tail = bugName.split(':')[2]
+            test_path = 'data/tests-list/' + group + '/' + group + '-bug-' + tail + '/' + 'tests-list.txt'
+            test_path = 'data/tests-package/' + group + '/' + group + '-bug-' + tail + '/' + 'tests-list.txt'
+            # test_path = 'data/tests-list-tian/' + group + '/' + group + '-bug-' + tail + '/' + 'tests-list.txt'
+            test_path = join(DATA_PATH,'tests-list-tian',group + '/' + group + '-bug-' + tail + '/' + 'tests-list.txt')
+            # test_path = 'data/tests-list-tian/' + group + '/' + group + '-bug-' + tail + '/' + 'tests-list.txt'
+            tests_list = []
+            if not os.path.exists(test_path):
+                # print("no tests-list!")
+                return
+            output = ''
+            output += "bugName: {}".format(bugName)
+
+            with open(test_path,'r') as f:
+                for line in f.readlines():
+                    tests_list.append(line.strip())
+
+            bug = client.bugs[bugName]
+            if client.bugs.is_installed(bug):
+                # print("the image is installed! :-)")
+                pass
+            else:
+                client.bugs.build(bug)
+                # client.bugs.download(bug)
+                # print("the image is not installed :'(")
+
+            # print("creating container...")
+            container = client.containers.provision(bug)
+            # print("container is ready")
+
+
+            output = patch_application_our(client, container,bug,tests_list,output)
+            print(output)
+            return output
+
+        except Exception as e:
+            print(e)
+            # continue
+        finally:
+            cmd = 'docker stop {}'.format(container.id)
+            out, e = shellGitCheckout(cmd)
+            client.shutdown()
+            os.killpg(process.pid, signal.SIGTERM)
+
+from bugzoo import Patch, Client
+
+def test_part(bug, container,  client, tests_list):
+    test_outcomes = []  # type: Dict[TestCase, TestOutcome]
+    failure_cases = []
+    failure = 0
+    total = len(tests_list)
+    if bug.name.startswith('manybugs:wireshark'):
+        cmd = ' ./autogen.sh && ./configure && make'
+        o = client.containers.exec(container=container, command=cmd, context='/experiment/src', time_limit=120)
+
+        cmd = 'sudo mkdir scenario && sudo cp -r src/ scenario'
+        o = client.containers.exec(container=container, command=cmd, context='/experiment', time_limit=120)
+    for test in tests_list:
+        cmd = './test.sh ' + test
+        out = client.containers.exec(container=container, command=cmd, context='/experiment/', time_limit=120)
+        # cmd = 'docker exec ' + container.id + ' ' + cmd
+        # o,e = shellGitCheckout(cmd)
+        # if test.expected_outcome != test_outcomes[test].passed:
+        if 'PASS' not in out.output or out.code != 0:
+            # o, e = shellGitCheckout(cmd)
+            out = client.containers.exec(container=container, command=cmd, context='/experiment/', time_limit=120)
+            if 'PASS' not in out.output or out.code != 0:
+                failure += 1
+                failure_cases.append(test)
+                test_outcomes.append(out.output)
+                break
+    return failure_cases, failure, total, test_outcomes
+
+# def test_all(bug, container,  client):
+#     test_outcomes = {}  # type: Dict[TestCase, TestOutcome]
+#     failure_cases = []
+#     failure = 0
+#     total = len(bug.tests._tests)
+#     for test in bug.tests:
+#         test_outcomes[test] = client.containers.test(container, test)
+#         # if test.expected_outcome != test_outcomes[test].passed:
+#         if test_outcomes[test].passed != True:
+#             failure += 1
+#             failure_cases.append(test.name)
+#             break
+#     return failure_cases, failure, total, test_outcomes
+
+# def get_diff_sos(bug, client, container):
+#     patch_list = []
+#     group = bug.split(':')[1]
+#     path = './data/manybugs_sos_patches/' + group + '/' + group + '-bug-' + bug.split(':')[2]
+#     for root, dirs, files in os.walk(path):
+#         for name in files:
+#             if name == 'sos.patch':
+#                 patch_list.append(os.path.join(root, name))
+#
+#     diff_files = []
+#     for patch_file in patch_list:
+#         unidiff = ''
+#         with open(patch_file,'r') as file:
+#             for line in file.readlines():
+#                 if group == 'wireshark':
+#                     if line.startswith('---'):
+#                         # begin = line.index('diffs') + 6
+#                         begin = line.index('a')
+#                         unidiff += '--- ' + line[begin+2:]
+#                     elif line.startswith('+++'):
+#                         # begin = line.index('diffs') + 6
+#                         begin = line.index('b')
+#                         unidiff += '+++ ' + line[begin+2:]
+#                     else:
+#                         unidiff += line
+#                 else:
+#                     if line.startswith('---'):
+#                         # begin = line.index('diffs') + 6
+#                         ends = line.index('_orig')
+#                         unidiff += line[:ends]+ ' \n'
+#                     else:
+#                         unidiff += line
+#             diff_files.append(unidiff.strip())
+#     return diff_files
+
+# def get_diff_our(bug, client, container):
+#     patch_list = []
+#     group = bug.split(':')[1]
+#     patches_path = './data/manybugs_anil_patches1/' + bug + '/patches'
+#     for root, dirs, files in os.walk(patches_path):
+#         for name in files:
+#             if name.endswith('.txt'):
+#                 patch_list.append(os.path.join(root, name))
+#
+#     diff_files = {}
+#     for patch_file in patch_list:
+#         unidiff = ''
+#         with open(patch_file,'r') as file:
+#             for line in file.readlines():
+#                 if line.startswith('---'):
+#                     begin = line.index('/diffs')
+#                     unidiff += '--- /experiment' + line[begin:]
+#                 elif line.startswith('+++'):
+#                     begin = line.index('diffs') + 6
+#                     ends = line.index('.c-')
+#                     unidiff += line[:4] + line[begin:ends] + '.c' + '\n'
+#                 else:
+#                     unidiff += line
+#             diff_files[patch_file.split('/')[-1]] = unidiff.strip()
+#     return diff_files
+
+# def get_diff(bug, client, container):
+#     patch_list = []
+#     path = './data/manybugs/' + bug + '/diffs'
+#     for root, dirs, files in os.walk(path):
+#         for name in files:
+#             if name.endswith('.patch'):
+#                 patch_list.append(os.path.join(root, name))
+#
+#     diff_files = []
+#     for patch_file in patch_list:
+#         unidiff = ''
+#         with open(patch_file,'r') as file:
+#             for line in file.readlines():
+#                 if line.startswith('---'):
+#                     begin = line.index('diffs') + 6
+#                     ends = line.index('.c-')
+#                     unidiff += line[:4] + line[begin:ends] + '.c' + '\n'
+#                     preFile = line[begin:line.index('\t')]
+#                 elif line.startswith('+++'):
+#                     begin = line.index('diffs') + 6
+#                     ends = line.index('.c-')
+#                     unidiff += line[:4] + line[begin:ends] + '.c' + '\n'
+#                 else:
+#                     unidiff += line
+#             cmd = 'cp '+ '../diffs/' + preFile + ' ' + preFile[:preFile.index('.c')+2]
+#             # cmd += ' & ' + 'make -j$(nproc)'
+#             client.containers.exec(container=container, command=cmd, context='/experiment/src')
+#             diff_files.append(unidiff.strip())
+#
+#     return diff_files
+
+# def patch_ours_test(bug, bugName, container, client, tests_list):
+#     patch_target = []
+#     path = './data/manybugs/' + bugName + '/diffs'
+#     for root, dirs, files in os.walk(path):
+#         for name in files:
+#             if name.endswith('.patch'):
+#                 file_path = os.path.join(root, name[:name.index('.patch')])
+#                 patch_target.append(file_path[file_path.index('diffs')+6:])
+#     if len(patch_target) != 1:
+#         print('exceed! ')
+#         return
+#     patched = []
+#     patched_path = './data/manybugs_tian_1/' + bugName + '/patched'
+#     for root, dirs, files in os.walk(patched_path):
+#         for name in files:
+#             if name.endswith('.c'):
+#                 patched.append(os.path.join(root, name))
+#     if len(patched) == 0:
+#         print('no patch! ')
+#         return
+#     times = 0
+#     for p in patched:
+#         if not p.endswith('origin.c'):
+#             continue
+#         cmd = 'docker cp ' + p + ' ' + container.id + ':/experiment/src/' + patch_target[0]
+#         o, e = shellGitCheckout(cmd)
+#         client.containers.build(container)
+#         failure_cases, failure, total, test_outcomes = test_part(bug, container, client, tests_list)
+#         if failure == 0:
+#             times += 1
+#             print('fixed by {}'.format(p),end=' ')
+#         else:
+#             print('{} failed'.format(failure_cases),end=' ')
+#     print('times:{}'.format(times),end=' ')
+#     if times > 0:
+#         print('success ')
+#     else:
+#         print('failure ')
+
+
+# def patch_application(client: Client, container: Container, diff_files: list) -> None:
+#     result = None
+#     for unidiff in diff_files:
+#         if len(unidiff) < 5:
+#             continue
+#         # first, we build a Patch object using a unified-format diff
+#         patch = Patch.from_unidiff(unidiff)
+#
+#         # we then attempt to apply the patch to the source code
+#         re = client.containers.patch(container, patch)
+#         if not re:
+#             return False
+#         else:
+#             result = True
+#     # finally, we rebuild the program inside the container
+#     client.containers.build(container)
+#
+#     return result
+
+def patch_application_our(client: Client, container: Container,bug,tests_list,output):
+    result = None
+    times = 0
+    # patches_path = './data/manybugs_tian_patches4topC/' + bug.name + '/patches'
+    patches_path = join(PATCHES_DIR,bug.name,'patches')
+    # patches_path = './data/manybugs_tian_patches4topC/' + bug.name + '/patches'
+    for root, dirs, files in os.walk(patches_path):
+        for name in files:
+            if name.endswith('.txt'):
+                patch_file = os.path.join(root, name)
+                unidiff = ''
+                with open(patch_file, 'r') as file:
+                    for line in file.readlines():
+                        if line.startswith('---'):
+                            begin = line.index('/diffs')
+                            origin_begin = line.index('diffs') + 6
+                            unidiff += '--- /experiment' + line[begin:]
+                            preFile = line[origin_begin:line.index('\n')]
+                        elif line.startswith('+++'):
+                            begin = line.index('diffs') + 6
+                            ends = line.index('.c-')
+                            unidiff += line[:4] + line[begin:ends] + '.c' + '\n'
+                        else:
+                            unidiff += line
+                    cmd = 'cp ' + '../diffs/' + preFile + ' ' + preFile[:preFile.index('.c') + 2]
+                    # cmd += ' & ' + 'make -j$(nproc)'
+                    client.containers.exec(container=container, command=cmd, context='/experiment/src')
+                    diff_files = unidiff.strip()
+
+                    if len(unidiff) < 5:
+                        continue
+                    # first, we build a Patch object using a unified-format diff
+                    patch = Patch.from_unidiff(unidiff)
+
+                    # we then attempt to apply the patch to the source code
+                    re = client.containers.patch(container, patch)
+                    if not re:
+                        output += ' {} patch failed'.format(name)
+                    else:
+                        result = True
+                        # finally, we rebuild the program inside the container
+                        patch_result = client.containers.build(container)
+                        if patch_result.successful:
+                            failure_cases, failure, total, test_outcomes = test_part(bug, container, client, tests_list)
+                            if failure == 0:
+                                times += 1
+                                output+= ' fixed by {}'.format(name)
+                            else:
+                                output += ' {}'.format(failure_cases)
+                        else:
+                            output += ' {}'.format('build error')
+    output += ' times:{}'.format(times)
+    if times > 0:
+        output += ' success'
+    else:
+        output += ' failure'
+    return output
+
+
+def getClient(port):
+    cmd = 'bash {} {}'.format(join(DATA_PATH, 'startBugzoo.sh'), port)
+
+    with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) as process:
+        # o,e = shellGitCheckout(cmd)
+        url = "http://127.0.0.1:{}".format(port)
+        timeout_connection = 3000
+        client = Client(url, timeout_connection=timeout_connection)
+        return client,process
+
+def validate():
+
+    buglist = ['manybugs:gmp:14166-14167', 'manybugs:gzip:2009-08-16-3fe0caeada-39a362ae9d',
+                 'manybugs:gzip:2009-10-09-1a085b1446-118a107f2d', 'manybugs:gzip:2010-02-19-3eb6091d69-884ef6d16c',
+                 'manybugs:libtiff:2005-12-21-3b848a7-3edb9cd', 'manybugs:libtiff:2006-03-03-a72cf60-0a36d7f',
+                 'manybugs:libtiff:2006-03-03-eec4c06-ee65c74', 'manybugs:libtiff:2007-07-13-09e8220-f2d989d',
+                 'manybugs:libtiff:2007-11-02-371336d-865f7b2', 'manybugs:libtiff:2009-02-05-764dbba-2e42d63',
+                 'manybugs:libtiff:2009-08-28-e8a47d4-023b6df', 'manybugs:libtiff:2010-11-27-eb326f9-eec7ec0',
+                 'manybugs:libtiff:2006-02-23-b2ce5d8-207c78a', 'manybugs:lighttpd:2661-2662',
+                 'manybugs:lighttpd:2254-2259', 'manybugs:lighttpd:2785-2786', 'manybugs:lighttpd:1948-1949',
+                 'manybugs:php:2011-12-10-74343ca506-52c36e60c4', 'manybugs:php:2011-04-02-70075bc84c-5a8c917c37',
+                 'manybugs:php:2011-03-25-8138f7de40-3acdca4703', 'manybugs:php:2011-12-04-1e6a82a1cf-dfa08dc325',
+                 'manybugs:php:2012-02-08-ff63c09e6f-6672171672', 'manybugs:php:2011-11-19-eeba0b5681-d3b20b4058',
+                 'manybugs:php:2011-04-07-77ed819430-efcb9a71cd', 'manybugs:php:2011-02-01-01745fa657-1f49902999',
+                 'manybugs:php:2012-03-12-7aefbf70a8-efc94f3115', 'manybugs:php:2011-10-15-0a1cc5f01c-05c5c8958e',
+                 'manybugs:php:2011-01-30-5bb0a44e06-1e91069eb4', 'manybugs:php:2011-02-01-fefe9fc5c7-0927309852',
+                 'manybugs:php:2011-02-27-e65d361fde-1d984a7ffd', 'manybugs:php:2011-03-19-5d0c948296-8deb11c0c3',
+                 'manybugs:php:2011-03-23-63673a533f-2adf58cfcf', 'manybugs:php:2011-04-06-187eb235fe-2e25ec9eb7',
+                 'manybugs:php:2011-04-09-db01e840c2-09b990f499', 'manybugs:php:2011-05-17-453c954f8a-daecb2c0f4',
+                 'manybugs:php:2011-05-24-b60f6774dc-1056c57fa9', 'manybugs:php:2011-10-16-1f78177e2b-d4ae4e79db',
+                 'manybugs:php:2011-10-31-2e5d5e5ac6-b5f15ef561', 'manybugs:php:2011-10-31-c4eb5f2387-2e5d5e5ac6',
+                 'manybugs:php:2011-11-01-ceac9dc490-9b0d73af1d', 'manybugs:php:2011-11-11-fcbfbea8d2-c1e510aea8',
+                 'manybugs:php:2011-11-15-236120d80e-fb37f3b20d', 'manybugs:php:2011-11-16-55acfdf7bd-3c7a573a2c',
+                 'manybugs:php:2011-11-22-ecc6c335c5-b548293b99', 'manybugs:php:2011-11-23-eca88d3064-db0888dfc1',
+                 'manybugs:php:2012-01-27-544e36dfff-acaf9c5227', 'manybugs:php:2012-01-30-9de5b6dc7c-4dc8b1ad11',
+                 'manybugs:php:2012-02-25-c1322d2505-cfa9c90b20', 'manybugs:php:2012-03-04-60dfd64bf2-34fe62619d',
+                 'manybugs:php:2012-03-08-0169020e49-cdc512afb3', 'manybugs:php:2012-03-11-3954743813-d4f05fbffc',
+                 'manybugs:php:2012-03-12-438a30f1e7-7337a901b7', 'manybugs:python:69223-69224',
+                 'manybugs:python:69368-69372', 'manybugs:python:70098-70101', 'manybugs:python:70056-70059',
+                 'manybugs:wireshark:37112-37111', 'manybugs:wireshark:37122-37123', 'manybugs:gmp:13420-13421',
+                 'manybugs:gzip:2009-09-26-a1d3d4019d-f17cbd13a1', 'manybugs:lighttpd:1913-1914',
+                 'manybugs:php:2011-11-19-51a4ae6576-bc810a443d', 'manybugs:php:2011-03-11-d890ece3fc-6e74d95f34',
+                 'manybugs:php:2011-11-19-eeba0b5681-f330c8ab4e', 'manybugs:php:2012-01-01-80dd931d40-7c3177e5ab']
+
+    port = 6000
+    bugList = []
+    for b in buglist:
+        if b == '.DS_Store':
+            continue
+        t = b, port
+        bugList.append(t)
+        if port == 6300:
+            port = 6000
+        port += 1
+
+    results = parallelRunMerge(patch_validate,bugList)
+
+    # #
+    with open(join(DATA_PATH, 'manyBugsResults'), 'w',
+              encoding='utf-8') as writeFile:
+        # if levelPatch == 0:
+        writeFile.write('\n'.join(results))
+    # patch_validate()
+    # pass

python/wmp00080.uni.lux.config.yml

@@ -0,0 +1,21 @@
+java:
+    8home:  /Library/Java/JavaVirtualMachines/jdk1.8.0_181.jdk/Contents/Home
+spinfer:
+    home:   /Users/anilkoyuncu/projects/fixminer/spinfer/spinfer.native
+    #home:   /Users/anil.koyuncu/projects/spinfer/spinfer.native
+dataset:
+#    home:   /Users/anil.koyuncu/projects/test/fixminer-core/python/data/gumInputLinux/
+    home:   /Users/anilkoyuncu/projects/fixminer/fixminer-core/python/data/gumInputLinux
+#    home:   /data/fixminer-core/python/data/gumInputLinux/
+    repo:   /Users/anil.koyuncu/projects/datasets
+
+coccinelle:
+    home:   /Users/anilkoyuncu/projects/fixminer/spinfer/statics
+    #home:   /Users/anil.koyuncu/projects/spinfer/statics
+
+
+fixminer:
+#    datapath: /Users/anil.koyuncu/projects/test/fixminer-core/python/data/
+    datapath: /Users/anilkoyuncu/projects/fixminer/fixminer-data/
+    projectType : c
+#    datapath: /data/fixminer-core/python/data/