signature:
def detect_fold_level ( self , prev_block , block ) :
    """Perfoms fold level detection for current block ( take previous block
 into account ) .
 : param prev _ block : previous block , None if ` block ` is the first block .
 : param block : block to analyse .
 : return : block fold level"""

implementation:
# Python is an indent based language so use indentation for folding
# makes sense but we restrict new regions to indentation after a ' : ' ,
# that way only the real logical blocks are displayed .
lvl = super ( PythonFoldDetector , self ) . detect_fold_level ( prev_block , block )
# cancel false indentation , indentation can only happen if there is
# ' : ' on the previous line
prev_lvl = TextBlockHelper . get_fold_lvl ( prev_block )
if prev_block and lvl > prev_lvl and not ( self . _strip_comments ( prev_block ) . endswith ( ':' ) ) :
    lvl = prev_lvl
lvl = self . _handle_docstrings ( block , lvl , prev_block )
lvl = self . _handle_imports ( block , lvl , prev_block )
return lvl


signature:
def setup_actions ( self ) :
    """Connects slots to signals"""

implementation:
self . actionOpen . triggered . connect ( self . on_open )
self . actionNew . triggered . connect ( self . on_new )
self . actionSave . triggered . connect ( self . on_save )
self . actionSave_as . triggered . connect ( self . on_save_as )
self . actionQuit . triggered . connect ( QtWidgets . QApplication . instance ( ) . quit )
self . tabWidget . current_changed . connect ( self . on_current_tab_changed )
self . tabWidget . last_tab_closed . connect ( self . on_last_tab_closed )
self . actionAbout . triggered . connect ( self . on_about )
self . actionRun . triggered . connect ( self . on_run )
self . interactiveConsole . process_finished . connect ( self . on_process_finished )
self . actionConfigure_run . triggered . connect ( self . on_configure_run )


signature:
def setup_editor ( self , editor ) :
    """Setup the python editor , run the server and connect a few signals .
 : param editor : editor to setup ."""

implementation:
editor . cursorPositionChanged . connect ( self . on_cursor_pos_changed )
try :
    m = editor . modes . get ( modes . GoToAssignmentsMode )
except KeyError :
    pass
else :
    assert isinstance ( m , modes . GoToAssignmentsMode )
    m . out_of_doc . connect ( self . on_goto_out_of_doc )


signature:
def open_file ( self , path , line = None ) :
    """Creates a new GenericCodeEdit , opens the requested file and adds it
 to the tab widget .
 : param path : Path of the file to open
 : return The opened editor if open succeeded ."""

implementation:
editor = None
if path :
    interpreter , pyserver , args = self . _get_backend_parameters ( )
    editor = self . tabWidget . open_document ( path , None , interpreter = interpreter , server_script = pyserver , args = args )
    if editor :
        self . setup_editor ( editor )
    self . recent_files_manager . open_file ( path )
    self . menu_recents . update_actions ( )
if line is not None :
    TextHelper ( self . tabWidget . current_widget ( ) ) . goto_line ( line )
return editor


signature:
def _get_backend_parameters ( self ) :
    """Gets the pyqode backend parameters ( interpreter and script ) ."""

implementation:
frozen = hasattr ( sys , 'frozen' )
interpreter = Settings ( ) . interpreter
if frozen :
    interpreter = None
pyserver = server . __file__ if interpreter is not None else 'server.exe'
args = [ ]
return interpreter , pyserver , args


signature:
def on_new ( self ) :
    """Add a new empty code editor to the tab widget"""

implementation:
interpreter , pyserver , args = self . _get_backend_parameters ( )
self . setup_editor ( self . tabWidget . create_new_document ( extension = '.py' , interpreter = interpreter , server_script = pyserver , args = args ) )
self . actionRun . setDisabled ( True )
self . actionConfigure_run . setDisabled ( True )


signature:
def on_open ( self ) :
    """Shows an open file dialog and open the file if the dialog was
 accepted ."""

implementation:
filename , filter = QtWidgets . QFileDialog . getOpenFileName ( self , 'Open' )
if filename :
    self . open_file ( filename )
self . actionRun . setEnabled ( True )
self . actionConfigure_run . setEnabled ( True )


signature:
def on_save_as ( self ) :
    """Save the current editor document as ."""

implementation:
path = self . tabWidget . current_widget ( ) . file . path
path = os . path . dirname ( path ) if path else ''
filename , filter = QtWidgets . QFileDialog . getSaveFileName ( self , 'Save' , path )
if filename :
    self . tabWidget . save_current ( filename )
    self . recent_files_manager . open_file ( filename )
    self . menu_recents . update_actions ( )
    self . actionRun . setEnabled ( True )
    self . actionConfigure_run . setEnabled ( True )
    self . _update_status_bar ( self . tabWidget . current_widget ( ) )


signature:
def setup_mnu_style ( self , editor ) :
    """setup the style menu for an editor tab"""

implementation:
menu = QtWidgets . QMenu ( 'Styles' , self . menuEdit )
group = QtWidgets . QActionGroup ( self )
self . styles_group = group
current_style = editor . syntax_highlighter . color_scheme . name
group . triggered . connect ( self . on_style_changed )
for s in sorted ( PYGMENTS_STYLES ) :
    a = QtWidgets . QAction ( menu )
    a . setText ( s )
    a . setCheckable ( True )
    if s == current_style :
        a . setChecked ( True )
    group . addAction ( a )
    menu . addAction ( a )
self . menuEdit . addMenu ( menu )


signature:
def setup_mnu_panels ( self , editor ) :
    """Setup the panels menu for the current editor .
 : param editor :"""

implementation:
for panel in editor . panels :
    if panel . dynamic :
        continue
    a = QtWidgets . QAction ( self . menuModes )
    a . setText ( panel . name )
    a . setCheckable ( True )
    a . setChecked ( panel . enabled )
    a . changed . connect ( self . on_panel_state_changed )
    a . panel = weakref . proxy ( panel )
    self . menuPanels . addAction ( a )


signature:
def on_current_tab_changed ( self ) :
    """Update action states when the current tab changed ."""

implementation:
self . menuEdit . clear ( )
self . menuModes . clear ( )
self . menuPanels . clear ( )
editor = self . tabWidget . current_widget ( )
self . menuEdit . setEnabled ( editor is not None )
self . menuModes . setEnabled ( editor is not None )
self . menuPanels . setEnabled ( editor is not None )
self . actionSave . setEnabled ( editor is not None )
self . actionSave_as . setEnabled ( editor is not None )
self . actionConfigure_run . setEnabled ( editor is not None )
self . actionRun . setEnabled ( editor is not None )
if editor is not None :
    self . setup_mnu_edit ( editor )
    self . setup_mnu_modes ( editor )
    self . setup_mnu_panels ( editor )
self . widgetOutline . set_editor ( editor )
self . _update_status_bar ( editor )


signature:
def on_run ( self ) :
    """Run the current current script"""

implementation:
filename = self . tabWidget . current_widget ( ) . file . path
wd = os . path . dirname ( filename )
args = Settings ( ) . get_run_config_for_file ( filename )
self . interactiveConsole . start_process ( Settings ( ) . interpreter , args = [ filename ] + args , cwd = wd )
self . dockWidget . show ( )
self . actionRun . setEnabled ( False )
self . actionConfigure_run . setEnabled ( False )


signature:
def on_goto_out_of_doc ( self , assignment ) :
    """Open the a new tab when goto goes out of the current document .
 : param assignment : Destination"""

implementation:
editor = self . open_file ( assignment . module_path )
if editor :
    TextHelper ( editor ) . goto_line ( assignment . line , assignment . column )


signature:
def calltips ( request_data ) :
    """Worker that returns a list of calltips .
 A calltips is a tuple made of the following parts :
 - module _ name : name of the module of the function invoked
 - call _ name : name of the function that is being called
 - params : the list of parameter names .
 - index : index of the current parameter
 - bracket _ start
 : returns tuple ( module _ name , call _ name , params )"""

implementation:
code = request_data [ 'code' ]
line = request_data [ 'line' ] + 1
column = request_data [ 'column' ]
path = request_data [ 'path' ]
# encoding = request _ data [ ' encoding ' ]
encoding = 'utf-8'
# use jedi to get call signatures
script = jedi . Script ( code , line , column , path , encoding )
signatures = script . call_signatures ( )
for sig in signatures :
    results = ( str ( sig . module_name ) , str ( sig . name ) , [ p . description for p in sig . params ] , sig . index , sig . bracket_start , column )
    # todo : add support for multiple signatures , for that we need a custom
    # widget for showing calltips .
    return results
return [ ]


signature:
def goto_assignments ( request_data ) :
    """Go to assignements worker ."""

implementation:
code = request_data [ 'code' ]
line = request_data [ 'line' ] + 1
column = request_data [ 'column' ]
path = request_data [ 'path' ]
# encoding = request _ data [ ' encoding ' ]
encoding = 'utf-8'
script = jedi . Script ( code , line , column , path , encoding )
try :
    definitions = script . goto_assignments ( )
except jedi . NotFoundError :
    pass
else :
    ret_val = [ ( d . module_path , d . line - 1 if d . line else None , d . column , d . full_name ) for d in definitions ]
    return ret_val


signature:
def defined_names ( request_data ) :
    """Returns the list of defined names for the document ."""

implementation:
global _old_definitions
ret_val = [ ]
path = request_data [ 'path' ]
toplvl_definitions = jedi . names ( request_data [ 'code' ] , path , 'utf-8' )
for d in toplvl_definitions :
    definition = _extract_def ( d , path )
    if d . type != 'import' :
        ret_val . append ( definition )
ret_val = [ d . to_dict ( ) for d in ret_val ]
return ret_val


signature:
def quick_doc ( request_data ) :
    """Worker that returns the documentation of the symbol under cursor ."""

implementation:
code = request_data [ 'code' ]
line = request_data [ 'line' ] + 1
column = request_data [ 'column' ]
path = request_data [ 'path' ]
# encoding = ' utf - 8'
encoding = 'utf-8'
script = jedi . Script ( code , line , column , path , encoding )
try :
    definitions = script . goto_definitions ( )
except jedi . NotFoundError :
    return [ ]
else :
    ret_val = [ d . docstring ( ) for d in definitions ]
    return ret_val


signature:
def run_pep8 ( request_data ) :
    """Worker that run the pep8 tool on the current editor text .
 : returns a list of tuples ( msg , msg _ type , line _ number )"""

implementation:
import pycodestyle
from pyqode . python . backend . pep8utils import CustomChecker
WARNING = 1
code = request_data [ 'code' ]
path = request_data [ 'path' ]
max_line_length = request_data [ 'max_line_length' ]
ignore_rules = request_data [ 'ignore_rules' ]
ignore_rules += [ 'W291' , 'W292' , 'W293' , 'W391' ]
pycodestyle . MAX_LINE_LENGTH = max_line_length
# setup our custom style guide with our custom checker which returns a list
# of strings instread of spitting the results at stdout
pep8style = pycodestyle . StyleGuide ( parse_argv = False , config_file = '' , checker_class = CustomChecker )
try :
    results = pep8style . input_file ( path , lines = code . splitlines ( True ) )
except Exception :
    _logger ( ) . exception ( 'Failed to run PEP8 analysis with data=%r' % request_data )
    return [ ]
else :
    messages = [ ]
    for line_number , offset , code , text , doc in results :
        if code in ignore_rules :
            continue
        messages . append ( ( '[PEP8] %s: %s' % ( code , text ) , WARNING , line_number - 1 ) )
    return messages


signature:
def run_pyflakes ( request_data ) :
    """Worker that run a frosted ( the fork of pyflakes ) code analysis on the
 current editor text ."""

implementation:
global prev_results
from pyflakes import checker
import _ast
WARNING = 1
ERROR = 2
ret_val = [ ]
code = request_data [ 'code' ]
path = request_data [ 'path' ]
encoding = request_data [ 'encoding' ]
if not encoding :
    encoding = 'utf-8'
if not path :
    path = os . path . join ( tempfile . gettempdir ( ) , 'temp.py' )
if not code :
    return [ ]
else : # First , compile into an AST and handle syntax errors .
    try :
        tree = compile ( code . encode ( encoding ) , path , "exec" , _ast . PyCF_ONLY_AST )
    except SyntaxError as value :
        msg = '[pyFlakes] %s' % value . args [ 0 ]
        ( lineno , offset , text ) = value . lineno - 1 , value . offset , value . text
        # If there ' s an encoding problem with the file , the text is None
        if text is None : # Avoid using msg , since for the only known case , it
        # contains a bogus message that claims the encoding the
        # file declared was unknown . s
            _logger ( ) . warning ( "[SyntaxError] %s: problem decoding source" , path )
        else :
            ret_val . append ( ( msg , ERROR , lineno ) )
    else : # Okay , it ' s syntactically valid . Now check it .
        w = checker . Checker ( tree , os . path . split ( path ) [ 1 ] )
        w . messages . sort ( key = lambda m : m . lineno )
        for message in w . messages :
            msg = "[pyFlakes] %s" % str ( message ) . split ( ':' ) [ - 1 ] . strip ( )
            line = message . lineno - 1
            status = WARNING if message . __class__ not in PYFLAKES_ERROR_MESSAGES else ERROR
            ret_val . append ( ( msg , status , line ) )
prev_results = ret_val
return ret_val


signature:
def icon_from_typename ( name , icon_type ) :
    """Returns the icon resource filename that corresponds to the given typename .
 : param name : name of the completion . Use to make the distinction between
 public and private completions ( using the count of starting ' _ ' )
 : pram typename : the typename reported by jedi
 : returns : The associate icon resource filename or None ."""

implementation:
ICONS = { 'CLASS' : ICON_CLASS , 'IMPORT' : ICON_NAMESPACE , 'STATEMENT' : ICON_VAR , 'FORFLOW' : ICON_VAR , 'FORSTMT' : ICON_VAR , 'WITHSTMT' : ICON_VAR , 'GLOBALSTMT' : ICON_VAR , 'MODULE' : ICON_NAMESPACE , 'KEYWORD' : ICON_KEYWORD , 'PARAM' : ICON_VAR , 'ARRAY' : ICON_VAR , 'INSTANCEELEMENT' : ICON_VAR , 'INSTANCE' : ICON_VAR , 'PARAM-PRIV' : ICON_VAR , 'PARAM-PROT' : ICON_VAR , 'FUNCTION' : ICON_FUNC , 'DEF' : ICON_FUNC , 'FUNCTION-PRIV' : ICON_FUNC_PRIVATE , 'FUNCTION-PROT' : ICON_FUNC_PROTECTED }
ret_val = None
icon_type = icon_type . upper ( )
# jedi 0.8 introduced NamedPart class , which have a string instead of being
# one
if hasattr ( name , "string" ) :
    name = name . string
if icon_type == "FORFLOW" or icon_type == "STATEMENT" :
    icon_type = "PARAM"
if icon_type == "PARAM" or icon_type == "FUNCTION" :
    if name . startswith ( "__" ) :
        icon_type += "-PRIV"
    elif name . startswith ( "_" ) :
        icon_type += "-PROT"
if icon_type in ICONS :
    ret_val = ICONS [ icon_type ]
elif icon_type :
    _logger ( ) . warning ( "Unimplemented completion icon_type: %s" , icon_type )
return ret_val


signature:
def complete ( code , line , column , path , encoding , prefix ) :
    """Completes python code using ` jedi ` _ .
 : returns : a list of completion ."""

implementation:
ret_val = [ ]
try :
    script = jedi . Script ( code , line + 1 , column , path , encoding )
    completions = script . completions ( )
    print ( 'completions: %r' % completions )
except jedi . NotFoundError :
    completions = [ ]
for completion in completions :
    ret_val . append ( { 'name' : completion . name , 'icon' : icon_from_typename ( completion . name , completion . type ) , 'tooltip' : completion . description } )
return ret_val


signature:
def make_python_patterns ( additional_keywords = [ ] , additional_builtins = [ ] ) :
    """Strongly inspired from idlelib . ColorDelegator . make _ pat"""

implementation:
kw = r"\b" + any ( "keyword" , kwlist + additional_keywords ) + r"\b"
kw_namespace = r"\b" + any ( "namespace" , kw_namespace_list ) + r"\b"
word_operators = r"\b" + any ( "operator_word" , wordop_list ) + r"\b"
builtinlist = [ str ( name ) for name in dir ( builtins ) if not name . startswith ( '_' ) ] + additional_builtins
for v in [ 'None' , 'True' , 'False' ] :
    builtinlist . remove ( v )
builtin = r"([^.'\"\\#]\b|^)" + any ( "builtin" , builtinlist ) + r"\b"
builtin_fct = any ( "builtin_fct" , [ r'_{2}[a-zA-Z_]*_{2}' ] )
comment = any ( "comment" , [ r"#[^\n]*" ] )
instance = any ( "instance" , [ r"\bself\b" , r"\bcls\b" ] )
decorator = any ( 'decorator' , [ r'@\w*' , r'.setter' ] )
number = any ( "number" , [ r"\b[+-]?[0-9]+[lLjJ]?\b" , r"\b[+-]?0[xX][0-9A-Fa-f]+[lL]?\b" , r"\b[+-]?0[oO][0-7]+[lL]?\b" , r"\b[+-]?0[bB][01]+[lL]?\b" , r"\b[+-]?[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?[jJ]?\b" ] )
sqstring = r"(\b[rRuU])?'[^'\\\n]*(\\.[^'\\\n]*)*'?"
dqstring = r'(\b[rRuU])?"[^"\\\n]*(\\.[^"\\\n]*)*"?'
uf_sqstring = r"(\b[rRuU])?'[^'\\\n]*(\\.[^'\\\n]*)*(\\)$(?!')$"
uf_dqstring = r'(\b[rRuU])?"[^"\\\n]*(\\.[^"\\\n]*)*(\\)$(?!")$'
sq3string = r"(\b[rRuU])?'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
dq3string = r'(\b[rRuU])?"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
uf_sq3string = r"(\b[rRuU])?'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(\\)?(?!''')$"
uf_dq3string = r'(\b[rRuU])?"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(\\)?(?!""")$'
string = any ( "string" , [ sq3string , dq3string , sqstring , dqstring ] )
ufstring1 = any ( "uf_sqstring" , [ uf_sqstring ] )
ufstring2 = any ( "uf_dqstring" , [ uf_dqstring ] )
ufstring3 = any ( "uf_sq3string" , [ uf_sq3string ] )
ufstring4 = any ( "uf_dq3string" , [ uf_dq3string ] )
return "|" . join ( [ instance , decorator , kw , kw_namespace , builtin , word_operators , builtin_fct , comment , ufstring1 , ufstring2 , ufstring3 , ufstring4 , string , number , any ( "SYNC" , [ r"\n" ] ) ] )


signature:
def _check_word_cursor ( self , tc = None ) :
    """Request a go to assignment .
 : param tc : Text cursor which contains the text that we must look for
 its assignment . Can be None to go to the text that is under
 the text cursor .
 : type tc : QtGui . QTextCursor"""

implementation:
if not tc :
    tc = TextHelper ( self . editor ) . word_under_cursor ( )
request_data = { 'code' : self . editor . toPlainText ( ) , 'line' : tc . blockNumber ( ) , 'column' : tc . columnNumber ( ) , 'path' : self . editor . file . path , 'encoding' : self . editor . file . encoding }
try :
    self . editor . backend . send_request ( workers . goto_assignments , request_data , on_receive = self . _on_results_available )
except NotRunning :
    pass


signature:
def _unique ( self , seq ) :
    """Not performant but works ."""

implementation:
# order preserving
checked = [ ]
for e in seq :
    present = False
    for c in checked :
        if str ( c ) == str ( e ) :
            present = True
            break
    if not present :
        checked . append ( e )
return checked


signature:
def read_bgen ( filepath , metafile_filepath = None , samples_filepath = None , verbose = True ) :
    r"""Read a given BGEN file .
 Parameters
 filepath : str
 A bgen file path .
 metafile _ filepath : str , optional
 If ` ` None ` ` , it will try to read the ` ` filepath + " . metadata " ` ` file . If this is
 not possible , it will create one . It tries to create one at
 ` ` filepath + " . metadata " ` ` . If that is also no possible , it tries to create one
 at a temporary folder .
 samples _ filepath : str , optional
 A sample file in ` gen format < https : / / goo . gl / bCzo7m > ` _ .
 If ` ` samples _ filepath ` ` is provided , sample ids are read from this file .
 Otherwise , it reads from the bgen file itself if possible . Defaults to ` ` None ` ` .
 verbose : bool , optional
 ` ` True ` ` to show progress ; ` ` False ` ` otherwise . Defaults to ` ` True ` ` .
 Returns
 variants : : class : ` dask . dataFrame . DataFrame `
 Variant position , chromosomes , rsids , etc .
 samples : : class : ` pandas . Series `
 Sample identifications .
 genotype : list
 List of genotypes .
 Examples
 . . doctest : :
 > > > from bgen _ reader import example _ files , read _ bgen
 > > > with example _ files ( " haplotypes . bgen " ) as filepath :
 . . . bgen = read _ bgen ( filepath , verbose = False )
 . . . variants = bgen [ " variants " ]
 . . . samples = bgen [ " samples " ]
 . . . v = variants . loc [ 0 ] . compute ( )
 . . . g = bgen [ " genotype " ] [ 0 ] . compute ( )
 . . . print ( v )
 . . . print ( samples )
 . . . print ( g [ " probs " ] [ 0 ] )
 id rsid chrom pos nalleles allele _ ids vaddr
 0 SNP1 RS1 1 1 2 A , G 102
 0 sample _ 0
 1 sample _ 1
 2 sample _ 2
 3 sample _ 3
 Name : id , dtype : object
 [1 . 0 . 1 . 0 . ]"""

implementation:
assert_file_exist ( filepath )
assert_file_readable ( filepath )
metafile_filepath = _get_valid_metafile_filepath ( filepath , metafile_filepath )
if not os . path . exists ( metafile_filepath ) :
    if verbose :
        print ( f"We will create the metafile `{metafile_filepath}`. This file will " "speed up further\nreads and only need to be created once. So, please, " "bear with me." )
    create_metafile ( filepath , metafile_filepath , verbose )
samples = get_samples ( filepath , samples_filepath , verbose )
variants = map_metadata ( filepath , metafile_filepath )
genotype = map_genotype ( filepath , metafile_filepath , verbose )
return dict ( variants = variants , samples = samples , genotype = genotype )


signature:
def _validateDirectives ( self , directiveList , checkFileName ) :
    if len ( directiveList ) == 0 :
        raise ParsingException ( "'{file}' does not contain any CHECK directives" . format ( file = checkFileName ) )
    from . import Directives
    """We should enforce for every CHECK - NOT and CHECK - NOT - L directive that the next directive ( if it exists ) is
 a CHECK or CHECK - L directive"""

implementation:
last = len ( directiveList ) - 1
supportedDirectives = [ Directives . Check , Directives . CheckLiteral ]
for ( index , directive ) in enumerate ( directiveList ) :
    if isA ( directive , [ Directives . CheckNot , Directives . CheckNotLiteral ] ) :
        if index < last :
            after = directiveList [ index + 1 ]
            if not isA ( after , supportedDirectives ) :
                requiredTypes = " or " . join ( [ "CHECK{suffix}" . format ( suffix = d . directiveToken ( ) ) for d in supportedDirectives ] )
                raise ParsingException ( "{directive} must have a {requiredTypes} directive after it instead of a {bad}" . format ( directive = directive , requiredTypes = requiredTypes , check = Directives . Check . directiveToken ( ) , bad = after ) )


signature:
def create_metafile ( bgen_filepath , metafile_filepath , verbose = True ) :
    r"""Create variants metadata file .
 Variants metadata file helps speed up subsequent reads of the associated
 bgen file .
 Parameters
 bgen _ filepath : str
 Bgen file path .
 metafile _ file : str
 Metafile file path .
 verbose : bool
 ` ` True ` ` to show progress ; ` ` False ` ` otherwise .
 Examples
 . . doctest : :
 > > > import os
 > > > from bgen _ reader import create _ metafile , example _ files
 > > > with example _ files ( " example . 32bits . bgen " ) as filepath :
 . . . folder = os . path . dirname ( filepath )
 . . . metafile _ filepath = os . path . join ( folder , filepath + " . metadata " )
 . . . try :
 . . . create _ metafile ( filepath , metafile _ filepath , verbose = False )
 . . . finally :
 . . . if os . path . exists ( metafile _ filepath ) :
 . . . os . remove ( metafile _ filepath )"""

implementation:
if verbose :
    verbose = 1
else :
    verbose = 0
bgen_filepath = make_sure_bytes ( bgen_filepath )
metafile_filepath = make_sure_bytes ( metafile_filepath )
assert_file_exist ( bgen_filepath )
assert_file_readable ( bgen_filepath )
if exists ( metafile_filepath ) :
    raise ValueError ( f"The file {metafile_filepath} already exists." )
with bgen_file ( bgen_filepath ) as bgen :
    nparts = _estimate_best_npartitions ( lib . bgen_nvariants ( bgen ) )
    metafile = lib . bgen_create_metafile ( bgen , metafile_filepath , nparts , verbose )
    if metafile == ffi . NULL :
        raise RuntimeError ( f"Error while creating metafile: {metafile_filepath}." )
    if lib . bgen_close_metafile ( metafile ) != 0 :
        raise RuntimeError ( f"Error while closing metafile: {metafile_filepath}." )


signature:
def match ( self , subsetLines , offsetOfSubset , fileName ) :
    """Search through lines for match .
 Raise an Exception if fail to match
 If match is succesful return the position the match was found"""

implementation:
for ( offset , l ) in enumerate ( subsetLines ) :
    column = l . find ( self . literal )
    if column != - 1 :
        truePosition = offset + offsetOfSubset
        _logger . debug ( 'Found match on line {}, col {}' . format ( str ( truePosition + 1 ) , column ) )
        _logger . debug ( 'Line is {}' . format ( l ) )
        self . matchLocation = CheckFileParser . FileLocation ( fileName , truePosition + 1 )
        return truePosition
# No Match found
self . failed = True
raise DirectiveException ( self )


signature:
def match ( self , subsetLines , offsetOfSubset , fileName ) :
    """Search through lines for match .
 Raise an Exception if a match"""

implementation:
for ( offset , l ) in enumerate ( subsetLines ) :
    for t in self . regex :
        m = t . Regex . search ( l )
        if m != None :
            truePosition = offset + offsetOfSubset
            _logger . debug ( 'Found match on line {}' . format ( str ( truePosition + 1 ) ) )
            _logger . debug ( 'Line is {}' . format ( l ) )
            self . failed = True
            self . matchLocation = CheckFileParser . FileLocation ( fileName , truePosition + 1 )
            raise DirectiveException ( self )


signature:
def isA ( instance , typeList ) :
    """Return true if ` ` instance ` ` is an instance of any the Directive
 types in ` ` typeList ` `"""

implementation: return any ( map ( lambda iType : isinstance ( instance , iType ) , typeList ) )


signature:
def _touch ( fname , mode = 0o666 , dir_fd = None , ** kwargs ) :
    """Touch a file .
 Credits to < https : / / stackoverflow . com / a / 1160227 > ."""

implementation:
flags = os . O_CREAT | os . O_APPEND
with os . fdopen ( os . open ( fname , flags = flags , mode = mode , dir_fd = dir_fd ) ) as f :
    os . utime ( f . fileno ( ) if os . utime in os . supports_fd else fname , dir_fd = None if os . supports_fd else dir_fd , ** kwargs , )


signature:
def allele_frequency ( expec ) :
    r"""Compute allele frequency from its expectation .
 Parameters
 expec : array _ like
 Allele expectations encoded as a samples - by - alleles matrix .
 Returns
 : class : ` numpy . ndarray `
 Allele frequencies encoded as a variants - by - alleles matrix .
 Examples
 . . doctest : :
 > > > from bgen _ reader import read _ bgen , example _ files
 > > > from bgen _ reader import allele _ expectation , allele _ frequency
 > > > # Download an example
 > > > example = example _ files ( " example . 32bits . bgen " )
 > > > filepath = example . filepath
 > > > bgen = read _ bgen ( filepath , verbose = False )
 > > > variants = bgen [ " variants " ]
 > > > samples = bgen [ " samples " ]
 > > > genotype = bgen [ " genotype " ]
 > > > variant = variants [ variants [ " rsid " ] = = " RSID _ 6 " ] . compute ( )
 > > > variant _ idx = variant . index . item ( )
 > > > p = genotype [ variant _ idx ] . compute ( ) [ " probs " ]
 > > > # For unphased genotypes only .
 > > > e = allele _ expectation ( bgen , variant _ idx )
 > > > f = allele _ frequency ( e )
 > > > alleles = variant [ " allele _ ids " ] . item ( ) . split ( " , " )
 > > > print ( alleles [ 0 ] + " : { } " . format ( f [ 0 ] ) )
 A : 229.23103218810434
 > > > print ( alleles [ 1 ] + " : { } " . format ( f [ 1 ] ) )
 G : 270.7689678118956
 > > > print ( variant )
 id rsid chrom pos nalleles allele _ ids vaddr
 4 SNPID _ 6 RSID _ 6 01 6000 2 A , G 19377
 > > > # Clean - up the example
 > > > example . close ( )"""

implementation:
expec = asarray ( expec , float )
if expec . ndim != 2 :
    raise ValueError ( "Expectation matrix must be bi-dimensional." )
ploidy = expec . shape [ - 1 ]
return expec . sum ( - 2 ) / ploidy


signature:
def compute_dosage ( expec , alt = None ) :
    r"""Compute dosage from allele expectation .
 Parameters
 expec : array _ like
 Allele expectations encoded as a samples - by - alleles matrix .
 alt : array _ like , optional
 Alternative allele index . If ` ` None ` ` , the allele having the minor
 allele frequency for the provided ` ` expec ` ` is used as the alternative .
 Defaults to ` ` None ` ` .
 Returns
 : class : ` numpy . ndarray `
 Dosage encoded as an array of size equal to the number of samples .
 Examples
 . . code - block : : python
 : caption : First a quick - start example .
 > > > from bgen _ reader import allele _ expectation , compute _ dosage
 > > > from bgen _ reader import example _ files , read _ bgen
 > > > # Download an example .
 > > > example = example _ files ( " example . 32bits . bgen " )
 > > > filepath = example . filepath
 > > > # Read the example .
 > > > bgen = read _ bgen ( filepath , verbose = False )
 > > > # Extract the allele expectations of the fourth variant .
 > > > variant _ idx = 3
 > > > e = allele _ expectation ( bgen , variant _ idx )
 > > > # Compute the dosage when considering the first allele
 > > > # as the reference / alternative one .
 > > > alt _ allele _ idx = 1
 > > > d = compute _ dosage ( e , alt = alt _ allele _ idx )
 > > > # Print the dosage of the first five samples only .
 > > > print ( d [ : 5 ] )
 [1.96185308 0.00982666 0.01745552 1.00347899 1.01153563]
 > > > # Clean - up the example
 > > > example . close ( )
 . . code - block : : python
 : caption : Genotype probabilities , allele expectations and frequencies .
 > > > from bgen _ reader import (
 . . . allele _ expectation ,
 . . . allele _ frequency ,
 . . . compute _ dosage ,
 . . . example _ files ,
 . . . read _ bgen ,
 > > > from pandas import DataFrame
 > > > from xarray import DataArray
 > > > # Download an example
 > > > example = example _ files ( " example . 32bits . bgen " )
 > > > filepath = example . filepath
 > > > # Open the bgen file .
 > > > bgen = read _ bgen ( filepath , verbose = False )
 > > > variants = bgen [ " variants " ]
 > > > genotype = bgen [ " genotype " ]
 > > > samples = bgen [ " samples " ]
 > > > variant _ idx = 3
 > > > variant = variants . loc [ variant _ idx ] . compute ( )
 > > > # Print the metadata of the fourth variant .
 > > > print ( variant )
 id rsid chrom pos nalleles allele _ ids vaddr
 3 SNPID _ 5 RSID _ 5 01 5000 2 A , G 16034
 > > > geno = bgen [ " genotype " ] [ variant _ idx ] . compute ( )
 > > > metageno = DataFrame ( { k : geno [ k ] for k in [ " ploidy " , " missing " ] } ,
 . . . index = samples )
 > > > metageno . index . name = " sample "
 > > > print ( metageno ) # doctest : + IGNORE _ EXCEPTION _ DETAIL , + NORMALIZE _ WHITESPACE
 ploidy missing
 sample
 sample _ 001 2 False
 sample _ 002 2 False
 sample _ 003 2 False
 sample _ 004 2 False
 sample _ 497 2 False
 sample _ 498 2 False
 sample _ 499 2 False
 sample _ 500 2 False
 < BLANKLINE >
 [500 rows x 2 columns ]
 > > > p = DataArray (
 . . . geno [ " probs " ] ,
 . . . name = " probability " ,
 . . . coords = { " sample " : samples } ,
 . . . dims = [ " sample " , " genotype " ] ,
 > > > # Print the genotype probabilities .
 > > > print ( p . to _ series ( ) . unstack ( level = - 1 ) ) # doctest : + IGNORE _ EXCEPTION _ DETAIL , + NORMALIZE _ WHITESPACE
 genotype 0 1 2
 sample
 sample _ 001 0.00488 0.02838 0.96674
 sample _ 002 0.99045 0.00928 0.00027
 sample _ 003 0.98932 0.00391 0.00677
 sample _ 004 0.00662 0.98328 0.01010
 sample _ 497 0.00137 0.01312 0.98550
 sample _ 498 0.00552 0.99423 0.00024
 sample _ 499 0.01266 0.01154 0.97580
 sample _ 500 0.00021 0.98431 0.01547
 < BLANKLINE >
 [500 rows x 3 columns ]
 > > > alleles = variant [ " allele _ ids " ] . item ( ) . split ( " , " )
 > > > e = DataArray (
 . . . allele _ expectation ( bgen , variant _ idx ) ,
 . . . name = " expectation " ,
 . . . coords = { " sample " : samples , " allele " : alleles } ,
 . . . dims = [ " sample " , " allele " ] ,
 > > > # Print the allele expectations .
 > > > print ( e . to _ series ( ) . unstack ( level = - 1 ) ) # doctest : + IGNORE _ EXCEPTION _ DETAIL , + NORMALIZE _ WHITESPACE
 allele A G
 sample
 sample _ 001 0.03815 1.96185
 sample _ 002 1.99017 0.00983
 sample _ 003 1.98254 0.01746
 sample _ 004 0.99652 1.00348
 sample _ 497 0.01587 1.98413
 sample _ 498 1.00528 0.99472
 sample _ 499 0.03687 1.96313
 sample _ 500 0.98474 1.01526
 < BLANKLINE >
 [500 rows x 2 columns ]
 > > > rsid = variant [ " rsid " ] . item ( )
 > > > chrom = variant [ " chrom " ] . item ( )
 > > > variant _ name = f " { chrom } : { rsid } "
 > > > f = DataFrame ( allele _ frequency ( e ) , columns = [ variant _ name ] , index = alleles )
 > > > f . index . name = " allele "
 > > > # Allele frequencies .
 > > > print ( f ) # doctest : + IGNORE _ EXCEPTION _ DETAIL , + NORMALIZE _ WHITESPACE
 01 : RSID _ 5
 allele
 A 305.97218
 G 194.02782
 > > > alt = f . idxmin ( ) . item ( )
 > > > alt _ idx = alleles . index ( alt )
 > > > d = compute _ dosage ( e , alt = alt _ idx ) . to _ series ( )
 > > > d = DataFrame ( d . values , columns = [ f " alt = { alt } " ] , index = d . index )
 > > > # Dosages when considering G as the alternative allele .
 > > > print ( d ) # doctest : + IGNORE _ EXCEPTION _ DETAIL , + NORMALIZE _ WHITESPACE
 alt = G
 sample
 sample _ 001 1.96185
 sample _ 002 0.00983
 sample _ 003 0.01746
 sample _ 004 1.00348
 sample _ 497 1.98413
 sample _ 498 0.99472
 sample _ 499 1.96313
 sample _ 500 1.01526
 < BLANKLINE >
 [500 rows x 1 columns ]
 > > > # Clean - up the example
 > > > example . close ( )"""

implementation:
if alt is None :
    return expec [ ... , - 1 ]
try :
    return expec [ : , alt ]
except NotImplementedError :
    alt = asarray ( alt , int )
    return asarray ( expec , float ) [ : , alt ]


signature:
def allele_expectation ( bgen , variant_idx ) :
    r"""Allele expectation .
 Compute the expectation of each allele from the genotype probabilities .
 Parameters
 bgen : bgen _ file
 Bgen file handler .
 variant _ idx : int
 Variant index .
 Returns
 : class : ` numpy . ndarray `
 Samples - by - alleles matrix of allele expectations .
 Note
 This function supports unphased genotypes only .
 Examples
 . . doctest : :
 > > > from bgen _ reader import allele _ expectation , example _ files , read _ bgen
 > > > from texttable import Texttable
 > > > # Download an example .
 > > > example = example _ files ( " example . 32bits . bgen " )
 > > > filepath = example . filepath
 > > > # Read the example .
 > > > bgen = read _ bgen ( filepath , verbose = False )
 > > > variants = bgen [ " variants " ]
 > > > samples = bgen [ " samples " ]
 > > > genotype = bgen [ " genotype " ]
 > > > genotype = bgen [ " genotype " ]
 > > > # This ` compute ` call will return a pandas data frame ,
 > > > variant = variants [ variants [ " rsid " ] = = " RSID _ 6 " ] . compute ( )
 > > > # from which we retrieve the variant index .
 > > > variant _ idx = variant . index . item ( )
 > > > print ( variant )
 id rsid chrom pos nalleles allele _ ids vaddr
 4 SNPID _ 6 RSID _ 6 01 6000 2 A , G 19377
 > > > genotype = bgen [ " genotype " ]
 > > > # Samples is a pandas series , and we retrieve the
 > > > # sample index from the sample name .
 > > > sample _ idx = samples [ samples = = " sample _ 005 " ] . index . item ( )
 > > > genotype = bgen [ " genotype " ]
 > > > # This ` compute ` call will return a dictionary from which
 > > > # we can get the probability matrix the corresponding
 > > > # variant .
 > > > p = genotype [ variant _ idx ] . compute ( ) [ " probs " ] [ sample _ idx ]
 > > > genotype = bgen [ " genotype " ]
 > > > # Allele expectation makes sense for unphased genotypes only ,
 > > > # which is the case here .
 > > > e = allele _ expectation ( bgen , variant _ idx ) [ sample _ idx ]
 > > > genotype = bgen [ " genotype " ]
 > > > alleles = variant [ " allele _ ids " ] . item ( ) . split ( " , " )
 > > > genotype = bgen [ " genotype " ]
 > > > # Print what we have got in a nice format .
 > > > table = Texttable ( )
 > > > table = table . add _ rows (
 . . . [ " " , " AA " , " AG " , " GG " , " E [ . ] " ] ,
 . . . [ " p " ] + list ( p ) + [ " na " ] ,
 . . . [ " # " + alleles [ 0 ] , 2 , 1 , 0 , e [ 0 ] ] ,
 . . . [ " # " + alleles [ 1 ] , 0 , 1 , 2 , e [ 1 ] ] ,
 > > > print ( table . draw ( ) )
 | | AA | AG | GG | E [ . ] |
 | p | 0.012 | 0.987 | 0.001 | na |
 | # A | 2 | 1 | 0 | 1.011 |
 | # G | 0 | 1 | 2 | 0.989 |
 > > > # Clean - up .
 > > > example . close ( )"""

implementation:
geno = bgen [ "genotype" ] [ variant_idx ] . compute ( )
if geno [ "phased" ] :
    raise ValueError ( "Allele expectation is define for unphased genotypes only." )
nalleles = bgen [ "variants" ] . loc [ variant_idx , "nalleles" ] . compute ( ) . item ( )
genotypes = get_genotypes ( geno [ "ploidy" ] , nalleles )
expec = [ ]
for i in range ( len ( genotypes ) ) :
    count = asarray ( genotypes_to_allele_counts ( genotypes [ i ] ) , float )
    n = count . shape [ 0 ]
    expec . append ( ( count . T * geno [ "probs" ] [ i , : n ] ) . sum ( 1 ) )
return stack ( expec , axis = 0 )


signature:
def find_libname ( self , name ) :
    """Try to infer the correct library name ."""

implementation:
names = [ "{}.lib" , "lib{}.lib" , "{}lib.lib" ]
names = [ n . format ( name ) for n in names ]
dirs = self . get_library_dirs ( )
for d in dirs :
    for n in names :
        if exists ( join ( d , n ) ) :
            return n [ : - 4 ]
msg = "Could not find the {} library." . format ( name )
raise ValueError ( msg )


signature:
def split ( self , X , y = None , groups = None ) :
    """Generate indices to split data into training and test set .
 Parameters
 X : array - like , of length n _ samples
 Training data , includes reaction ' s containers
 y : array - like , of length n _ samples
 The target variable for supervised learning problems .
 groups : array - like , with shape ( n _ samples , )
 Group labels for the samples used while splitting the dataset into
 train / test set .
 Yields
 train : ndarray
 The training set indices for that split .
 test : ndarray
 The testing set indices for that split ."""

implementation:
X , y , groups = indexable ( X , y , groups )
cgrs = [ ~ r for r in X ]
structure_condition = defaultdict ( set )
for structure , condition in zip ( cgrs , groups ) :
    structure_condition [ structure ] . add ( condition )
train_data = defaultdict ( list )
test_data = [ ]
for n , ( structure , condition ) in enumerate ( zip ( cgrs , groups ) ) :
    train_data [ condition ] . append ( n )
    if len ( structure_condition [ structure ] ) > 1 :
        test_data . append ( n )
for condition , indexes in train_data . items ( ) :
    test_index = [ index for index in indexes if index in test_data ]
    if test_index :
        train_index = [ i for cond , ind in train_data . items ( ) if cond != condition for i in ind ]
        yield array ( train_index ) , array ( test_index )


signature:
def molconvert_chemaxon ( data ) :
    """molconvert wrapper
 : param data : buffer or string or path to file
 : return : array of molecules of reactions"""

implementation:
if isinstance ( data , Path ) :
    with data . open ( 'rb' ) as f :
        data = f . read ( )
elif isinstance ( data , StringIO ) :
    data = data . read ( ) . encode ( )
elif isinstance ( data , BytesIO ) :
    data = data . read ( )
elif hasattr ( data , 'read' ) : # check if data is open ( filename , mode )
    data = data . read ( )
    if isinstance ( data , str ) :
        data = data . encode ( )
elif isinstance ( data , str ) :
    data = data . encode ( )
elif not isinstance ( data , bytes ) :
    raise ValueError ( 'invalid input' )
try :
    p = run ( [ 'molconvert' , '-g' , 'mrv' ] , input = data , stdout = PIPE )
except FileNotFoundError as e :
    raise ConfigurationError from e
if p . returncode != 0 :
    raise ConfigurationError ( p . stderr . decode ( ) )
with BytesIO ( p . stdout ) as f , MRVread ( f ) as r :
    return iter2array ( r )


signature:
def fit ( self , X , y = None ) :
    """Fit distance - based AD .
 Parameters
 X : array - like or sparse matrix , shape ( n _ samples , n _ features )
 The input samples . Use ` ` dtype = np . float32 ` ` for maximum
 efficiency .
 Returns
 self : object
 Returns self ."""

implementation:
# Check data
X = check_array ( X )
self . tree = BallTree ( X , leaf_size = self . leaf_size , metric = self . metric )
dist_train = self . tree . query ( X , k = 2 ) [ 0 ]
if self . threshold == 'auto' :
    self . threshold_value = 0.5 * sqrt ( var ( dist_train [ : , 1 ] ) ) + mean ( dist_train [ : , 1 ] )
elif self . threshold == 'cv' :
    if y is None :
        raise ValueError ( "Y must be specified to find the optimal threshold." )
    y = check_array ( y , accept_sparse = 'csc' , ensure_2d = False , dtype = None )
    self . threshold_value = 0
    score = 0
    Y_pred , Y_true , AD = [ ] , [ ] , [ ]
    cv = KFold ( n_splits = 5 , random_state = 1 , shuffle = True )
    for train_index , test_index in cv . split ( X ) :
        x_train = safe_indexing ( X , train_index )
        x_test = safe_indexing ( X , test_index )
        y_train = safe_indexing ( y , train_index )
        y_test = safe_indexing ( y , test_index )
        data_test = safe_indexing ( dist_train [ : , 1 ] , test_index )
        if self . reg_model is None :
            reg_model = RandomForestRegressor ( n_estimators = 500 , random_state = 1 ) . fit ( x_train , y_train )
        else :
            reg_model = clone ( self . reg_model ) . fit ( x_train , y_train )
        Y_pred . append ( reg_model . predict ( x_test ) )
        Y_true . append ( y_test )
        AD . append ( data_test )
    AD_ = unique ( hstack ( AD ) )
    for z in AD_ :
        AD_new = hstack ( AD ) <= z
        if self . score == 'ba_ad' :
            val = balanced_accuracy_score_with_ad ( Y_true = hstack ( Y_true ) , Y_pred = hstack ( Y_pred ) , AD = AD_new )
        elif self . score == 'rmse_ad' :
            val = rmse_score_with_ad ( Y_true = hstack ( Y_true ) , Y_pred = hstack ( Y_pred ) , AD = AD_new )
        if val >= score :
            score = val
            self . threshold_value = z
else :
    self . threshold_value = self . threshold
return self


signature:
def predict_proba ( self , X ) :
    """Returns the value of the nearest neighbor from the training set .
 Parameters
 X : array - like or sparse matrix , shape ( n _ samples , n _ features )
 The input samples . Internally , it will be converted to
 ` ` dtype = np . float32 ` ` and if a sparse matrix is provided
 to a sparse ` ` csr _ matrix ` ` .
 Returns
 y : array , shape ( n _ samples , )"""

implementation:
# Check is fit had been called
check_is_fitted ( self , [ 'tree' ] )
# Check data
X = check_array ( X )
return self . tree . query ( X ) [ 0 ] . flatten ( )


signature:
def predict ( self , X ) :
    """Predict if a particular sample is an outlier or not .
 Parameters
 X : array - like or sparse matrix , shape ( n _ samples , n _ features )
 The input samples . Internally , it will be converted to
 ` ` dtype = np . float32 ` ` and if a sparse matrix is provided
 to a sparse ` ` csr _ matrix ` ` .
 Returns
 y : array , shape ( n _ samples , )
 For each observations , tells whether or not ( True or False ) it should
 be considered as an inlier according to the fitted model ."""

implementation:
# Check is fit had been called
check_is_fitted ( self , [ 'tree' ] )
# Check data
X = check_array ( X )
return self . tree . query ( X ) [ 0 ] . flatten ( ) <= self . threshold_value


signature:
def fit ( self , X , y = None ) :
    """Learning is to find the inverse matrix for X and calculate the threshold .
 Parameters
 X : array - like or sparse matrix , shape ( n _ samples , n _ features )
 The input samples . Use ` ` dtype = np . float32 ` ` for maximum
 efficiency .
 y : array - like , shape = [ n _ samples ] or [ n _ samples , n _ outputs ]
 The target values ( real numbers in regression ) .
 Returns
 self : object"""

implementation:
# Check that X have correct shape
X = check_array ( X )
self . inverse_influence_matrix = self . __make_inverse_matrix ( X )
if self . threshold == 'auto' :
    self . threshold_value = 3 * ( 1 + X . shape [ 1 ] ) / X . shape [ 0 ]
elif self . threshold == 'cv' :
    if y is None :
        raise ValueError ( "Y must be specified to find the optimal threshold." )
    y = check_array ( y , accept_sparse = 'csc' , ensure_2d = False , dtype = None )
    self . threshold_value = 0
    score = 0
    Y_pred , Y_true , AD = [ ] , [ ] , [ ]
    cv = KFold ( n_splits = 5 , random_state = 1 , shuffle = True )
    for train_index , test_index in cv . split ( X ) :
        x_train = safe_indexing ( X , train_index )
        x_test = safe_indexing ( X , test_index )
        y_train = safe_indexing ( y , train_index )
        y_test = safe_indexing ( y , test_index )
        if self . reg_model is None :
            reg_model = RandomForestRegressor ( n_estimators = 500 , random_state = 1 ) . fit ( x_train , y_train )
        else :
            reg_model = clone ( self . reg_model ) . fit ( x_train , y_train )
        Y_pred . append ( reg_model . predict ( x_test ) )
        Y_true . append ( y_test )
        ad_model = self . __make_inverse_matrix ( x_train )
        AD . append ( self . __find_leverages ( x_test , ad_model ) )
    AD_ = unique ( hstack ( AD ) )
    for z in AD_ :
        AD_new = hstack ( AD ) <= z
        if self . score == 'ba_ad' :
            val = balanced_accuracy_score_with_ad ( Y_true = hstack ( Y_true ) , Y_pred = hstack ( Y_pred ) , AD = AD_new )
        elif self . score == 'rmse_ad' :
            val = rmse_score_with_ad ( Y_true = hstack ( Y_true ) , Y_pred = hstack ( Y_pred ) , AD = AD_new )
        if val >= score :
            score = val
            self . threshold_value = z
else :
    self . threshold_value = self . threshold
return self


signature:
def predict_proba ( self , X ) :
    """Predict the distances for X to center of the training set .
 Parameters
 X : array - like or sparse matrix , shape ( n _ samples , n _ features )
 The input samples . Internally , it will be converted to
 ` ` dtype = np . float32 ` ` and if a sparse matrix is provided
 to a sparse ` ` csr _ matrix ` ` .
 Returns
 leverages : array of shape = [ n _ samples ]
 The objects distances to center of the training set ."""

implementation:
# Check is fit had been called
check_is_fitted ( self , [ 'inverse_influence_matrix' ] )
# Check that X have correct shape
X = check_array ( X )
return self . __find_leverages ( X , self . inverse_influence_matrix )


signature:
def predict ( self , X ) :
    """Predict inside or outside AD for X .
 Parameters
 X : array - like or sparse matrix , shape ( n _ samples , n _ features )
 The input samples . Internally , it will be converted to
 ` ` dtype = np . float32 ` ` and if a sparse matrix is provided
 to a sparse ` ` csr _ matrix ` ` .
 Returns
 ad : array of shape = [ n _ samples ]
 Array contains True ( reaction in AD ) and False ( reaction residing outside AD ) ."""

implementation:
# Check is fit had been called
check_is_fitted ( self , [ 'inverse_influence_matrix' ] )
# Check that X have correct shape
X = check_array ( X )
return self . __find_leverages ( X , self . inverse_influence_matrix ) <= self . threshold_value


signature:
def get_feature_names ( self ) :
    """Get feature names .
 Returns
 feature _ names : list of strings
 Names of the features produced by transform ."""

implementation: return [ 'temperature' , 'pressure' ] + [ f'solvent.{x}' for x in range ( 1 , self . max_solvents + 1 ) ] + [ f'solvent_amount.{x}' for x in range ( 1 , self . max_solvents + 1 ) ]


signature:
def fit ( self , X , y = None ) :
    """Find min and max values of every feature .
 Parameters
 X : { array - like , sparse matrix } , shape ( n _ samples , n _ features )
 The training input samples .
 y : Ignored
 not used , present for API consistency by convention .
 Returns
 self : object"""

implementation:
# Check that X have correct shape
X = check_array ( X )
self . _x_min = X . min ( axis = 0 )
# axis = 0 will find the minimum values by columns ( for each feature )
self . _x_max = X . max ( axis = 0 )
# axis = 0 will find the minimum values by columns ( for each feature )
return self


signature:
def predict ( self , X ) :
    """Predict if a particular sample is an outlier or not .
 Parameters
 X : array - like or sparse matrix , shape ( n _ samples , n _ features )
 The input samples . Internally , it will be converted to
 ` ` dtype = np . float32 ` ` and if a sparse matrix is provided
 to a sparse ` ` csr _ matrix ` ` .
 Returns
 is _ inlier : array , shape ( n _ samples , )
 For each observations , tells whether or not ( True or False ) it should
 be considered as an inlier according to the fitted model ."""

implementation:
# Check is fit had been called
check_is_fitted ( self , [ '_x_min' , '_x_max' ] )
# Input validation
X = check_array ( X )
return ( ( X - self . _x_min ) . min ( axis = 1 ) >= 0 ) & ( ( self . _x_max - X ) . min ( axis = 1 ) >= 0 )


implementation:
X , y , groups = indexable ( X , y , groups )
cgrs = [ ~ r for r in X ]
condition_structure = defaultdict ( set )
for structure , condition in zip ( cgrs , groups ) :
    condition_structure [ condition ] . add ( structure )
train_data = defaultdict ( list )
test_data = [ ]
for n , ( structure , condition ) in enumerate ( zip ( cgrs , groups ) ) :
    train_data [ structure ] . append ( n )
    if len ( condition_structure [ condition ] ) > 1 :
        test_data . append ( n )
if self . n_splits > len ( train_data ) :
    raise ValueError ( "Cannot have number of splits n_splits=%d greater" " than the number of transformations: %d." % ( self . n_splits , len ( train_data ) ) )
structures_weight = sorted ( ( ( x , len ( y ) ) for x , y in train_data . items ( ) ) , key = lambda x : x [ 1 ] , reverse = True )
fold_mean_size = len ( cgrs ) // self . n_splits
if structures_weight [ 0 ] [ 1 ] > fold_mean_size :
    warning ( 'You have transformation that greater fold size' )
for idx in range ( self . n_repeats ) :
    train_folds = [ [ ] for _ in range ( self . n_splits ) ]
    for structure , structure_length in structures_weight :
        if self . shuffle :
            check_random_state ( self . random_state ) . shuffle ( train_folds )
        for fold in train_folds [ : - 1 ] :
            if len ( fold ) + structure_length <= fold_mean_size :
                fold . extend ( train_data [ structure ] )
                break
            else :
                roulette_param = ( structure_length - fold_mean_size + len ( fold ) ) / structure_length
                if random ( ) > roulette_param :
                    fold . extend ( train_data [ structure ] )
                    break
        else :
            train_folds [ - 1 ] . extend ( train_data [ structure ] )
    test_folds = [ [ ] for _ in range ( self . n_splits ) ]
    for test , train in zip ( test_folds , train_folds ) :
        for index in train :
            if index in test_data :
                test . append ( index )
    for i in range ( self . n_splits ) :
        train_index = [ ]
        for fold in train_folds [ : i ] :
            train_index . extend ( fold )
        for fold in train_folds [ i + 1 : ] :
            train_index . extend ( fold )
        test_index = test_folds [ i ]
        yield array ( train_index ) , array ( test_index )


signature:
def fit ( self , x , y = None ) :
    """Do nothing and return the estimator unchanged
 This method is just there to implement the usual API and hence work in pipelines ."""

implementation:
if self . _dtype is not None :
    iter2array ( x , dtype = self . _dtype )
else :
    iter2array ( x )
return self


signature:
def finalize ( self ) :
    """finalize partial fitting procedure"""

implementation:
if self . __head_less :
    warn ( f'{self.__class__.__name__} configured to head less mode. finalize unusable' )
elif not self . __head_generate :
    warn ( f'{self.__class__.__name__} already finalized or fitted' )
elif not self . __head_dict :
    raise NotFittedError ( f'{self.__class__.__name__} instance is not fitted yet' )
else :
    if self . remove_rare_ratio :
        self . __clean_head ( * self . __head_rare )
        self . __prepare_header ( )
        self . __head_rare = None
    self . __head_generate = False


signature:
def _reset ( self ) :
    """Reset internal data - dependent state .
 _ _ init _ _ parameters are not touched ."""

implementation:
if not self . __head_less :
    if not self . __head_generate :
        self . __head_generate = True
    if self . __head_dict :
        self . __head_dump = self . __head_dict = None
    if self . __head_rare is not None :
        self . __head_rare = None
    self . delete_work_path ( )


implementation:
if self . __head_less :
    raise AttributeError ( f'{self.__class__.__name__} instance configured to head less mode' )
elif not self . __head_dict :
    raise NotFittedError ( f'{self.__class__.__name__} instance is not fitted yet' )
return list ( self . __head_dict . values ( ) )


signature:
def fit ( self , x , y = None ) :
    """Compute the header ."""

implementation:
x = iter2array ( x , dtype = ( MoleculeContainer , CGRContainer ) )
if self . __head_less :
    warn ( f'{self.__class__.__name__} configured to head less mode. fit unusable' )
    return self
self . _reset ( )
self . __prepare ( x )
return self


signature:
def fit ( self , X ) :
    """Fit structure - based AD . The training model memorizes the unique set of reaction signature .
 Parameters
 X : after read rdf file
 Returns
 self : object"""

implementation:
X = iter2array ( X , dtype = ReactionContainer )
self . _train_signatures = { self . __get_signature ( x ) for x in X }
return self


signature:
def predict ( self , X ) :
    """Reaction is considered belonging to model ’ s AD
 if its reaction signature coincides with ones used in training set .
 Parameters
 X : after read rdf file
 Returns
 self : array contains True ( reaction in AD ) and False ( reaction residing outside AD ) ."""

implementation:
check_is_fitted ( self , [ '_train_signatures' ] )
X = iter2array ( X , dtype = ReactionContainer )
return array ( [ self . __get_signature ( x ) in self . _train_signatures for x in X ] )


signature:
def __parser ( expression ) :
    """adopted from Paul McGuire example . http : / / pyparsing . wikispaces . com / file / view / fourFn . py"""

implementation:
expr_stack = [ ]
def push_first ( strg , loc , toks ) :
    expr_stack . append ( toks [ 0 ] )
def push_u_minus ( strg , loc , toks ) :
    if toks and toks [ 0 ] == '-' :
        expr_stack . append ( 'unary -' )
point = Literal ( '.' )
_e = CaselessLiteral ( 'E' )
fnumber = Combine ( Word ( '+-' + nums , nums ) + Optional ( point + Optional ( Word ( nums ) ) ) + Optional ( _e + Word ( '+-' + nums , nums ) ) )
ident = Word ( alphas , alphas + nums + '_$' )
plus = Literal ( "+" )
minus = Literal ( "-" )
mult = Literal ( "*" )
div = Literal ( "/" )
lpar = Literal ( "(" ) . suppress ( )
rpar = Literal ( ")" ) . suppress ( )
addop = plus | minus
multop = mult | div
expop = Literal ( "^" )
_pi = CaselessLiteral ( "PI" )
x = CaselessLiteral ( "X" )
expr = Forward ( )
atom = ( Optional ( "-" ) + ( x | _pi | _e | fnumber | ident + lpar + expr + rpar ) . setParseAction ( push_first ) | ( lpar + expr . suppress ( ) + rpar ) ) . setParseAction ( push_u_minus )
factor = Forward ( )
factor << atom + ZeroOrMore ( ( expop + factor ) . setParseAction ( push_first ) )
term = factor + ZeroOrMore ( ( multop + factor ) . setParseAction ( push_first ) )
expr << term + ZeroOrMore ( ( addop + term ) . setParseAction ( push_first ) )
expr . parseString ( expression )
return expr_stack


signature:
def from_int ( data ) :
    """: params data : integer
 : returns : proquint made from input data
 : type data : int
 : rtype : string"""

implementation:
if not isinstance ( data , int ) and not isinstance ( data , long ) :
    raise TypeError ( 'Input must be integer' )
res = [ ]
while data > 0 or not res :
    for j in range ( 5 ) :
        if not j % 2 :
            res += CONSONANTS [ ( data & 0xf ) ]
            data >>= 4
        else :
            res += VOWELS [ ( data & 0x3 ) ]
            data >>= 2
    if data > 0 :
        res += '-'
res . reverse ( )
return '' . join ( res )


signature:
def to_int ( data ) :
    """: params data : proquint
 : returns : proquint decoded into an integer
 : type data : string
 : rtype : int"""

implementation:
if not isinstance ( data , basestring ) :
    raise TypeError ( 'Input must be string' )
res = 0
for part in data . split ( '-' ) :
    if len ( part ) != 5 :
        raise ValueError ( 'Malformed proquint' )
    for j in range ( 5 ) :
        try :
            if not j % 2 :
                res <<= 4
                res |= CONSONANTS . index ( part [ j ] )
            else :
                res <<= 2
                res |= VOWELS . index ( part [ j ] )
        except ValueError :
            raise ValueError ( 'Unknown character \'{!s}\' in proquint' . format ( part [ j ] ) )
return res


signature:
def get_or_create_shared_key ( cls , force_new = False ) :
    """Create a shared public / private key pair for certificate pushing ,
 if the settings allow ."""

implementation:
if force_new :
    with transaction . atomic ( ) :
        SharedKey . objects . filter ( current = True ) . update ( current = False )
        key = Key ( )
        return SharedKey . objects . create ( public_key = key , private_key = key , current = True )
# create a new shared key if one doesn ' t exist
try :
    return SharedKey . objects . get ( current = True )
except SharedKey . DoesNotExist :
    key = Key ( )
    return SharedKey . objects . create ( public_key = key , private_key = key , current = True )


signature:
def _self_referential_fk ( klass_model ) :
    """Return whether this model has a self ref FK , and the name for the field"""

implementation:
for f in klass_model . _meta . concrete_fields :
    if f . related_model :
        if issubclass ( klass_model , f . related_model ) :
            return f . attname
return None


signature:
def get_or_create_current_instance ( cls ) :
    """Get the instance model corresponding to the current system , or create a new
 one if the system is new or its properties have changed ( e . g . OS from upgrade ) ."""

implementation:
# on Android , platform . platform ( ) barfs , so we handle that safely here
try :
    plat = platform . platform ( )
except :
    plat = "Unknown (Android?)"
kwargs = { "platform" : plat , "hostname" : platform . node ( ) , "sysversion" : sys . version , "database" : DatabaseIDModel . get_or_create_current_database_id ( ) , "db_path" : os . path . abspath ( settings . DATABASES [ 'default' ] [ 'NAME' ] ) , "system_id" : os . environ . get ( "MORANGO_SYSTEM_ID" , "" ) , }
# try to get the MAC address , but exclude it if it was a fake ( random ) address
mac = uuid . getnode ( )
if ( mac >> 40 ) % 2 == 0 : # 8th bit ( of 48 bits , from left ) is 1 if MAC is fake
    hashable_identifier = "{}:{}" . format ( kwargs [ 'database' ] . id , mac )
    kwargs [ "node_id" ] = hashlib . sha1 ( hashable_identifier . encode ( 'utf-8' ) ) . hexdigest ( ) [ : 20 ]
else :
    kwargs [ "node_id" ] = ""
# do within transaction so we only ever have 1 current instance ID
with transaction . atomic ( ) :
    InstanceIDModel . objects . filter ( current = True ) . update ( current = False )
    obj , created = InstanceIDModel . objects . get_or_create ( ** kwargs )
    obj . current = True
    obj . save ( )
return obj , created


signature:
def _deserialize_store_model ( self , fk_cache ) :
    """When deserializing a store model , we look at the deleted flags to know if we should delete the app model .
 Upon loading the app model in memory we validate the app models fields , if any errors occurs we follow
 foreign key relationships to see if the related model has been deleted to propagate that deletion to the target app model .
 We return :
 None = > if the model was deleted successfully
 model = > if the model validates successfully"""

implementation:
klass_model = _profile_models [ self . profile ] [ self . model_name ]
# if store model marked as deleted , attempt to delete in app layer
if self . deleted : # if hard deleted , propagate to related models
    if self . hard_deleted :
        try :
            klass_model . objects . get ( id = self . id ) . delete ( hard_delete = True )
        except klass_model . DoesNotExist :
            pass
    else :
        klass_model . objects . filter ( id = self . id ) . delete ( )
    return None
else : # load model into memory
    app_model = klass_model . deserialize ( json . loads ( self . serialized ) )
    app_model . _morango_source_id = self . source_id
    app_model . _morango_partition = self . partition
    app_model . _morango_dirty_bit = False
    try : # validate and return the model
        app_model . cached_clean_fields ( fk_cache )
        return app_model
    except exceptions . ValidationError as e :
        logger . warn ( "Validation error for {model} with id {id}: {error}" . format ( model = klass_model . __name__ , id = app_model . id , error = e ) )
        # check FKs in store to see if any of those models were deleted or hard _ deleted to propagate to this model
        fk_ids = [ getattr ( app_model , field . attname ) for field in app_model . _meta . fields if isinstance ( field , ForeignKey ) ]
        for fk_id in fk_ids :
            try :
                st_model = Store . objects . get ( id = fk_id )
                if st_model . deleted : # if hard deleted , propagate to store model
                    if st_model . hard_deleted :
                        app_model . _update_hard_deleted_models ( )
                    app_model . _update_deleted_models ( )
                    return None
            except Store . DoesNotExist :
                pass
        raise e


signature:
def serialize ( self ) :
    """All concrete fields of the ` ` SyncableModel ` ` subclass , except for those specifically blacklisted , are returned in a dict ."""

implementation:
# NOTE : code adapted from https : / / github . com / django / django / blob / master / django / forms / models . py # L75
opts = self . _meta
data = { }
for f in opts . concrete_fields :
    if f . attname in self . morango_fields_not_to_serialize :
        continue
    if f . attname in self . _morango_internal_fields_not_to_serialize :
        continue
    # case if model is morango mptt
    if f . attname in getattr ( self , '_internal_mptt_fields_not_to_serialize' , '_internal_fields_not_to_serialize' ) :
        continue
    if hasattr ( f , 'value_from_object_json_compatible' ) :
        data [ f . attname ] = f . value_from_object_json_compatible ( self )
    else :
        data [ f . attname ] = f . value_from_object ( self )
return data


signature:
def deserialize ( cls , dict_model ) :
    """Returns an unsaved class object based on the valid properties passed in ."""

implementation:
kwargs = { }
for f in cls . _meta . concrete_fields :
    if f . attname in dict_model :
        kwargs [ f . attname ] = dict_model [ f . attname ]
return cls ( ** kwargs )


signature:
def get_default ( self ) :
    """Returns the default value for this field ."""

implementation:
if self . has_default ( ) :
    if callable ( self . default ) :
        default = self . default ( )
        if isinstance ( default , uuid . UUID ) :
            return default . hex
        return default
    if isinstance ( self . default , uuid . UUID ) :
        return self . default . hex
    return self . default
return None


signature:
def calculate_uuid ( self ) :
    """Should return a 32 - digit hex string for a UUID that is calculated as a function of a set of fields from the model ."""

implementation:
# raise an error if no inputs to the UUID calculation were specified
if self . uuid_input_fields is None :
    raise NotImplementedError ( """You must define either a 'uuid_input_fields' attribute
                (with a tuple of field names) or override the 'calculate_uuid' method, on models
                that inherit from UUIDModelMixin. If you want a fully random UUID, you can set
                'uuid_input_fields' to the string 'RANDOM'.""" )
# if the UUID has been set to be random , return a random UUID
if self . uuid_input_fields == "RANDOM" :
    return uuid . uuid4 ( ) . hex
# if we got this far , uuid _ input _ fields should be a tuple
assert isinstance ( self . uuid_input_fields , tuple ) , "'uuid_input_fields' must either be a tuple or the string 'RANDOM'"
# calculate the input to the UUID function
hashable_input_vals = [ ]
for field in self . uuid_input_fields :
    new_value = getattr ( self , field )
    if new_value :
        hashable_input_vals . append ( str ( new_value ) )
hashable_input = ":" . join ( hashable_input_vals )
# if all the values were falsey , just return a random UUID , to avoid collisions
if not hashable_input :
    return uuid . uuid4 ( ) . hex
# compute the UUID as a function of the input values
return sha2_uuid ( hashable_input )


signature:
def add_to_deleted_models ( sender , instance = None , * args , ** kwargs ) :
    """Whenever a model is deleted , we record its ID in a separate model for tracking purposes . During serialization , we will mark
 the model as deleted in the store ."""

implementation:
if issubclass ( sender , SyncableModel ) :
    instance . _update_deleted_models ( )


signature:
def make_request ( self , url , method = 'get' , headers = None , data = None , callback = None , errors = STRICT , verify = False , timeout = None , ** params ) :
    """Reusable method for performing requests .
 : param url - URL to request
 : param method - request method , default is ' get '
 : param headers - request headers
 : param data - post data
 : param callback - callback to be applied to response ,
 default callback will parse response as json object .
 : param errors - specifies communication errors handling mode , possible
 values are :
 * strict ( default ) - throw an error as soon as one
 occurred
 * graceful - ignore certain errors , e . g . EmptyResponse
 * ignore - ignore all errors and return a result in
 any case .
 NOTE that it DOES NOT mean that no
 exceptions can be
 raised from this method , it mostly ignores
 communication
 related errors .
 * None or empty string equals to default
 : param verify - whether or not to verify SSL cert , default to False
 : param timeout - the timeout of the request in second , default to None
 : param params - additional query parameters for request"""

implementation:
error_modes = ( STRICT , GRACEFUL , IGNORE )
error_mode = errors or GRACEFUL
if error_mode . lower ( ) not in error_modes :
    raise ValueError ( 'Possible values for errors argument are: %s' % ',' . join ( error_modes ) )
if callback is None :
    callback = self . _default_resp_callback
request = getattr ( requests , method . lower ( ) )
log . debug ( '* Request URL: %s' % url )
log . debug ( '* Request method: %s' % method )
log . debug ( '* Request query params: %s' % params )
log . debug ( '* Request headers: %s' % headers )
log . debug ( '* Request timeout: %s' % timeout )
r = request ( url , headers = headers , data = data , verify = verify , timeout = timeout , params = params )
log . debug ( '* r.url: %s' % r . url )
try :
    r . raise_for_status ( )
    return callback ( r )
except Exception as e :
    return self . _with_error_handling ( r , e , error_mode , self . response_format )


signature:
def _with_error_handling ( resp , error , mode , response_format ) :
    """Static method for error handling .
 : param resp - API response
 : param error - Error thrown
 : param mode - Error mode
 : param response _ format - XML or json"""

implementation:
def safe_parse ( r ) :
    try :
        return APIWrapper . _parse_resp ( r , response_format )
    except ( ValueError , SyntaxError ) as ex :
        log . error ( ex )
        r . parsed = None
        return r
if isinstance ( error , requests . HTTPError ) :
    if resp . status_code == 400 : # It means that request parameters were rejected by the server ,
    # so we need to enrich standard error message
    # with ' ValidationErrors '
    # from the response
        resp = safe_parse ( resp )
        if resp . parsed is not None :
            parsed_resp = resp . parsed
            messages = [ ]
            if response_format == 'xml' and parsed_resp . find ( './ValidationErrors' ) is not None :
                messages = [ e . find ( './Message' ) . text for e in parsed_resp . findall ( './ValidationErrors/ValidationErrorDto' ) ]
            elif response_format == 'json' and 'ValidationErrors' in parsed_resp :
                messages = [ e [ 'Message' ] for e in parsed_resp [ 'ValidationErrors' ] ]
            error = requests . HTTPError ( '%s: %s' % ( error , '\n\t' . join ( messages ) ) , response = resp )
    elif resp . status_code == 429 :
        error = requests . HTTPError ( '%sToo many requests in the last minute.' % error , response = resp )
if STRICT == mode :
    raise error
elif GRACEFUL == mode :
    if isinstance ( error , EmptyResponse ) : # Empty response is returned by the API occasionally ,
    # in this case it makes sense to ignore it and retry .
        log . warning ( error )
        resp . parsed = None
        return resp
    elif isinstance ( error , requests . HTTPError ) : # Ignoring ' Too many requests ' error ,
    # since subsequent retries will come after a delay .
        if resp . status_code == 429 : # Too many requests
            log . warning ( error )
            return safe_parse ( resp )
        else :
            raise error
    else :
        raise error
else : # ignore everything , just log it and return whatever response we
# have
    log . error ( error )
    return safe_parse ( resp )


signature:
def poll ( self , url , initial_delay = 2 , delay = 1 , tries = 20 , errors = STRICT , is_complete_callback = None , ** params ) :
    """Poll the URL
 : param url - URL to poll , should be returned by ' create _ session ' call
 : param initial _ delay - specifies how many seconds to wait before the first poll
 : param delay - specifies how many seconds to wait between the polls
 : param tries - number of polls to perform
 : param errors - errors handling mode , see corresponding parameter in ' make _ request ' method
 : param params - additional query params for each poll request"""

implementation:
time . sleep ( initial_delay )
poll_response = None
if is_complete_callback == None :
    is_complete_callback = self . _default_poll_callback
for n in range ( tries ) :
    poll_response = self . make_request ( url , headers = self . _headers ( ) , errors = errors , ** params )
    if is_complete_callback ( poll_response ) :
        return poll_response
    else :
        time . sleep ( delay )
if STRICT == errors :
    raise ExceededRetries ( "Failed to poll within {0} tries." . format ( tries ) )
else :
    return poll_response


signature:
def _default_poll_callback ( self , poll_resp ) :
    """Checks the condition in poll response to determine if it is complete
 and no subsequent poll requests should be done ."""

implementation:
if poll_resp . parsed is None :
    return False
success_list = [ 'UpdatesComplete' , True , 'COMPLETE' ]
status = None
if self . response_format == 'xml' :
    status = poll_resp . parsed . find ( './Status' ) . text
elif self . response_format == 'json' :
    status = poll_resp . parsed . get ( 'Status' , poll_resp . parsed . get ( 'status' ) )
if status is None :
    raise RuntimeError ( 'Unable to get poll response status.' )
return status in success_list


signature:
def _fsic_queuing_calc ( fsic1 , fsic2 ) :
    """We set the lower counter between two same instance ids .
 If an instance _ id exists in one fsic but not the other we want to give that counter a value of 0.
 : param fsic1 : dictionary containing ( instance _ id , counter ) pairs
 : param fsic2 : dictionary containing ( instance _ id , counter ) pairs
 : return ` ` dict ` ` of fsics to be used in queueing the correct records to the buffer"""

implementation: return { instance : fsic2 . get ( instance , 0 ) for instance , counter in six . iteritems ( fsic1 ) if fsic2 . get ( instance , 0 ) < counter }


signature:
def _serialize_into_store ( profile , filter = None ) :
    """Takes data from app layer and serializes the models into the store ."""

implementation:
# ensure that we write and retrieve the counter in one go for consistency
current_id = InstanceIDModel . get_current_instance_and_increment_counter ( )
with transaction . atomic ( ) : # create Q objects for filtering by prefixes
    prefix_condition = None
    if filter :
        prefix_condition = functools . reduce ( lambda x , y : x | y , [ Q ( _morango_partition__startswith = prefix ) for prefix in filter ] )
    # filter through all models with the dirty bit turned on
    syncable_dict = _profile_models [ profile ]
    for ( _ , klass_model ) in six . iteritems ( syncable_dict ) :
        new_store_records = [ ]
        new_rmc_records = [ ]
        klass_queryset = klass_model . objects . filter ( _morango_dirty_bit = True )
        if prefix_condition :
            klass_queryset = klass_queryset . filter ( prefix_condition )
        store_records_dict = Store . objects . in_bulk ( id_list = klass_queryset . values_list ( 'id' , flat = True ) )
        for app_model in klass_queryset :
            try :
                store_model = store_records_dict [ app_model . id ]
                # if store record dirty and app record dirty , append store serialized to conflicting data
                if store_model . dirty_bit :
                    store_model . conflicting_serialized_data = store_model . serialized + "\n" + store_model . conflicting_serialized_data
                    store_model . dirty_bit = False
                # set new serialized data on this store model
                ser_dict = json . loads ( store_model . serialized )
                ser_dict . update ( app_model . serialize ( ) )
                store_model . serialized = DjangoJSONEncoder ( ) . encode ( ser_dict )
                # create or update instance and counter on the record max counter for this store model
                RecordMaxCounter . objects . update_or_create ( defaults = { 'counter' : current_id . counter } , instance_id = current_id . id , store_model_id = store_model . id )
                # update last saved bys for this store model
                store_model . last_saved_instance = current_id . id
                store_model . last_saved_counter = current_id . counter
                # update deleted flags in case it was previously deleted
                store_model . deleted = False
                store_model . hard_deleted = False
                # update this model
                store_model . save ( )
            except KeyError :
                kwargs = { 'id' : app_model . id , 'serialized' : DjangoJSONEncoder ( ) . encode ( app_model . serialize ( ) ) , 'last_saved_instance' : current_id . id , 'last_saved_counter' : current_id . counter , 'model_name' : app_model . morango_model_name , 'profile' : app_model . morango_profile , 'partition' : app_model . _morango_partition , 'source_id' : app_model . _morango_source_id , }
                # check if model has FK pointing to it and add the value to a field on the store
                self_ref_fk = _self_referential_fk ( klass_model )
                if self_ref_fk :
                    self_ref_fk_value = getattr ( app_model , self_ref_fk )
                    kwargs . update ( { '_self_ref_fk' : self_ref_fk_value or '' } )
                # create store model and record max counter for the app model
                new_store_records . append ( Store ( ** kwargs ) )
                new_rmc_records . append ( RecordMaxCounter ( store_model_id = app_model . id , instance_id = current_id . id , counter = current_id . counter ) )
        # bulk create store and rmc records for this class
        Store . objects . bulk_create ( new_store_records )
        RecordMaxCounter . objects . bulk_create ( new_rmc_records )
        # set dirty bit to false for all instances of this model
        klass_queryset . update ( update_dirty_bit_to = False )
    # get list of ids of deleted models
    deleted_ids = DeletedModels . objects . filter ( profile = profile ) . values_list ( 'id' , flat = True )
    # update last _ saved _ bys and deleted flag of all deleted store model instances
    deleted_store_records = Store . objects . filter ( id__in = deleted_ids )
    deleted_store_records . update ( dirty_bit = False , deleted = True , last_saved_instance = current_id . id , last_saved_counter = current_id . counter )
    # update rmcs counters for deleted models that have our instance id
    RecordMaxCounter . objects . filter ( instance_id = current_id . id , store_model_id__in = deleted_ids ) . update ( counter = current_id . counter )
    # get a list of deleted model ids that don ' t have an rmc for our instance id
    new_rmc_ids = deleted_store_records . exclude ( recordmaxcounter__instance_id = current_id . id ) . values_list ( "id" , flat = True )
    # bulk create these new rmcs
    RecordMaxCounter . objects . bulk_create ( [ RecordMaxCounter ( store_model_id = r_id , instance_id = current_id . id , counter = current_id . counter ) for r_id in new_rmc_ids ] )
    # clear deleted models table for this profile
    DeletedModels . objects . filter ( profile = profile ) . delete ( )
    # handle logic for hard deletion models
    hard_deleted_ids = HardDeletedModels . objects . filter ( profile = profile ) . values_list ( 'id' , flat = True )
    hard_deleted_store_records = Store . objects . filter ( id__in = hard_deleted_ids )
    hard_deleted_store_records . update ( hard_deleted = True , serialized = '{}' , conflicting_serialized_data = '' )
    HardDeletedModels . objects . filter ( profile = profile ) . delete ( )
    # update our own database max counters after serialization
    if not filter :
        DatabaseMaxCounter . objects . update_or_create ( instance_id = current_id . id , partition = "" , defaults = { 'counter' : current_id . counter } )
    else :
        for f in filter :
            DatabaseMaxCounter . objects . update_or_create ( instance_id = current_id . id , partition = f , defaults = { 'counter' : current_id . counter } )


signature:
def _deserialize_from_store ( profile ) :
    """Takes data from the store and integrates into the application ."""

implementation:
# we first serialize to avoid deserialization merge conflicts
_serialize_into_store ( profile )
fk_cache = { }
with transaction . atomic ( ) :
    syncable_dict = _profile_models [ profile ]
    excluded_list = [ ]
    # iterate through classes which are in foreign key dependency order
    for model_name , klass_model in six . iteritems ( syncable_dict ) : # handle cases where a class has a single FK reference to itself
        self_ref_fk = _self_referential_fk ( klass_model )
        query = Q ( model_name = klass_model . morango_model_name )
        for klass in klass_model . morango_model_dependencies :
            query |= Q ( model_name = klass . morango_model_name )
        if self_ref_fk :
            clean_parents = Store . objects . filter ( dirty_bit = False , profile = profile ) . filter ( query ) . char_ids_list ( )
            dirty_children = Store . objects . filter ( dirty_bit = True , profile = profile ) . filter ( Q ( _self_ref_fk__in = clean_parents ) | Q ( _self_ref_fk = '' ) ) . filter ( query )
            # keep iterating until size of dirty _ children is 0
            while len ( dirty_children ) > 0 :
                for store_model in dirty_children :
                    try :
                        app_model = store_model . _deserialize_store_model ( fk_cache )
                        if app_model :
                            with mute_signals ( signals . pre_save , signals . post_save ) :
                                app_model . save ( update_dirty_bit_to = False )
                        # we update a store model after we have deserialized it to be able to mark it as a clean parent
                        store_model . dirty_bit = False
                        store_model . save ( update_fields = [ 'dirty_bit' ] )
                    except exceptions . ValidationError : # if the app model did not validate , we leave the store dirty bit set
                        excluded_list . append ( store_model . id )
                # update lists with new clean parents and dirty children
                clean_parents = Store . objects . filter ( dirty_bit = False , profile = profile ) . filter ( query ) . char_ids_list ( )
                dirty_children = Store . objects . filter ( dirty_bit = True , profile = profile , _self_ref_fk__in = clean_parents ) . filter ( query )
        else : # array for holding db values from the fields of each model for this class
            db_values = [ ]
            fields = klass_model . _meta . fields
            for store_model in Store . objects . filter ( model_name = model_name , profile = profile , dirty_bit = True ) :
                try :
                    app_model = store_model . _deserialize_store_model ( fk_cache )
                    # if the model was not deleted add its field values to the list
                    if app_model :
                        for f in fields :
                            value = getattr ( app_model , f . attname )
                            db_value = f . get_db_prep_value ( value , connection )
                            db_values . append ( db_value )
                except exceptions . ValidationError : # if the app model did not validate , we leave the store dirty bit set
                    excluded_list . append ( store_model . id )
            if db_values : # number of rows to update
                num_of_rows = len ( db_values ) // len ( fields )
                # create ' % s ' placeholders for a single row
                placeholder_tuple = tuple ( [ '%s' for _ in range ( len ( fields ) ) ] )
                # create list of the ' % s ' tuple placeholders based on number of rows to update
                placeholder_list = [ str ( placeholder_tuple ) for _ in range ( num_of_rows ) ]
                with connection . cursor ( ) as cursor :
                    DBBackend . _bulk_insert_into_app_models ( cursor , klass_model . _meta . db_table , fields , db_values , placeholder_list )
    # clear dirty bit for all store models for this profile except for models that did not validate
    Store . objects . exclude ( id__in = excluded_list ) . filter ( profile = profile , dirty_bit = True ) . update ( dirty_bit = False )


signature:
def _queue_into_buffer ( transfersession ) :
    """Takes a chunk of data from the store to be put into the buffer to be sent to another morango instance ."""

implementation:
last_saved_by_conditions = [ ]
filter_prefixes = Filter ( transfersession . filter )
server_fsic = json . loads ( transfersession . server_fsic )
client_fsic = json . loads ( transfersession . client_fsic )
if transfersession . push :
    fsics = _fsic_queuing_calc ( client_fsic , server_fsic )
else :
    fsics = _fsic_queuing_calc ( server_fsic , client_fsic )
# if fsics are identical or receiving end has newer data , then there is nothing to queue
if not fsics :
    return
# create condition for all push FSICs where instance _ ids are equal , but internal counters are higher than FSICs counters
for instance , counter in six . iteritems ( fsics ) :
    last_saved_by_conditions += [ "(last_saved_instance = '{0}' AND last_saved_counter > {1})" . format ( instance , counter ) ]
if fsics :
    last_saved_by_conditions = [ _join_with_logical_operator ( last_saved_by_conditions , 'OR' ) ]
partition_conditions = [ ]
# create condition for filtering by partitions
for prefix in filter_prefixes :
    partition_conditions += [ "partition LIKE '{}%'" . format ( prefix ) ]
if filter_prefixes :
    partition_conditions = [ _join_with_logical_operator ( partition_conditions , 'OR' ) ]
# combine conditions
fsic_and_partition_conditions = _join_with_logical_operator ( last_saved_by_conditions + partition_conditions , 'AND' )
# filter by profile
where_condition = _join_with_logical_operator ( [ fsic_and_partition_conditions , "profile = '{}'" . format ( transfersession . sync_session . profile ) ] , 'AND' )
# execute raw sql to take all records that match condition , to be put into buffer for transfer
with connection . cursor ( ) as cursor :
    queue_buffer = """INSERT INTO {outgoing_buffer}
                        (model_uuid, serialized, deleted, last_saved_instance, last_saved_counter, hard_deleted,
                         model_name, profile, partition, source_id, conflicting_serialized_data, transfer_session_id, _self_ref_fk)
                        SELECT id, serialized, deleted, last_saved_instance, last_saved_counter, hard_deleted, model_name, profile, partition, source_id, conflicting_serialized_data, '{transfer_session_id}', _self_ref_fk
                        FROM {store} WHERE {condition}""" . format ( outgoing_buffer = Buffer . _meta . db_table , transfer_session_id = transfersession . id , condition = where_condition , store = Store . _meta . db_table )
    cursor . execute ( queue_buffer )
    # take all record max counters that are foreign keyed onto store models , which were queued into the buffer
    queue_rmc_buffer = """INSERT INTO {outgoing_rmcb}
                            (instance_id, counter, transfer_session_id, model_uuid)
                            SELECT instance_id, counter, '{transfer_session_id}', store_model_id
                            FROM {record_max_counter} AS rmc
                            INNER JOIN {outgoing_buffer} AS buffer ON rmc.store_model_id = buffer.model_uuid
                            WHERE buffer.transfer_session_id = '{transfer_session_id}'
                            """ . format ( outgoing_rmcb = RecordMaxCounterBuffer . _meta . db_table , transfer_session_id = transfersession . id , record_max_counter = RecordMaxCounter . _meta . db_table , outgoing_buffer = Buffer . _meta . db_table )
    cursor . execute ( queue_rmc_buffer )


signature:
def _dequeue_into_store ( transfersession ) :
    """Takes data from the buffers and merges into the store and record max counters ."""

implementation:
with connection . cursor ( ) as cursor :
    DBBackend . _dequeuing_delete_rmcb_records ( cursor , transfersession . id )
    DBBackend . _dequeuing_delete_buffered_records ( cursor , transfersession . id )
    current_id = InstanceIDModel . get_current_instance_and_increment_counter ( )
    DBBackend . _dequeuing_merge_conflict_buffer ( cursor , current_id , transfersession . id )
    DBBackend . _dequeuing_merge_conflict_rmcb ( cursor , transfersession . id )
    DBBackend . _dequeuing_update_rmcs_last_saved_by ( cursor , current_id , transfersession . id )
    DBBackend . _dequeuing_delete_mc_rmcb ( cursor , transfersession . id )
    DBBackend . _dequeuing_delete_mc_buffer ( cursor , transfersession . id )
    DBBackend . _dequeuing_insert_remaining_buffer ( cursor , transfersession . id )
    DBBackend . _dequeuing_insert_remaining_rmcb ( cursor , transfersession . id )
    DBBackend . _dequeuing_delete_remaining_rmcb ( cursor , transfersession . id )
    DBBackend . _dequeuing_delete_remaining_buffer ( cursor , transfersession . id )
if getattr ( settings , 'MORANGO_DESERIALIZE_AFTER_DEQUEUING' , True ) :
    _deserialize_from_store ( transfersession . sync_session . profile )


signature:
def max_parameter_substitution ( ) :
    """SQLite has a limit on the max number of variables allowed for parameter substitution . This limit is usually 999 , but
 can be compiled to a different number . This function calculates what the max is for the sqlite version running on the device .
 We use the calculated value to chunk our SQL bulk insert statements when deserializing from the store to the app layer ."""

implementation:
if os . path . isfile ( SQLITE_VARIABLE_FILE_CACHE ) :
    return
conn = sqlite3 . connect ( ':memory:' )
low = 1
high = 1000
# hard limit for SQLITE _ MAX _ VARIABLE _ NUMBER < http : / / www . sqlite . org / limits . html >
conn . execute ( 'CREATE TABLE T1 (id C1)' )
while low < high - 1 :
    guess = ( low + high ) // 2
    try :
        statement = 'select * from T1 where id in (%s)' % ',' . join ( [ '?' for _ in range ( guess ) ] )
        values = [ i for i in range ( guess ) ]
        conn . execute ( statement , values )
    except sqlite3 . DatabaseError as ex :
        if 'too many SQL variables' in str ( ex ) :
            high = guess
        else :
            raise
    else :
        low = guess
conn . close ( )
with open ( SQLITE_VARIABLE_FILE_CACHE , 'w' ) as file :
    file . write ( str ( low ) )


signature:
def authenticate_credentials ( self , userargs , password , request = None ) :
    """Authenticate the userargs and password against Django auth backends .
 The " userargs " string may be just the username , or a querystring - encoded set of params ."""

implementation:
credentials = { 'password' : password }
if "=" not in userargs : # if it doesn ' t seem to be in querystring format , just use it as the username
    credentials [ get_user_model ( ) . USERNAME_FIELD ] = userargs
else : # parse out the user args from querystring format into the credentials dict
    for arg in userargs . split ( "&" ) :
        key , val = arg . split ( "=" )
        credentials [ key ] = val
# authenticate the user via Django ' s auth backends
user = authenticate ( ** credentials )
if user is None :
    raise exceptions . AuthenticationFailed ( 'Invalid credentials.' )
if not user . is_active :
    raise exceptions . AuthenticationFailed ( 'User inactive or deleted.' )
return ( user , None )


signature:
def _multiple_self_ref_fk_check ( class_model ) :
    """We check whether a class has more than 1 FK reference to itself ."""

implementation:
self_fk = [ ]
for f in class_model . _meta . concrete_fields :
    if f . related_model in self_fk :
        return True
    if f . related_model == class_model :
        self_fk . append ( class_model )
return False


signature:
def add_syncable_models ( ) :
    """Per profile , adds each model to a dictionary mapping the morango model name to its model class .
 We sort by ForeignKey dependencies to safely sync data ."""

implementation:
import django . apps
from morango . models import SyncableModel
from morango . manager import SyncableModelManager
from morango . query import SyncableModelQuerySet
model_list = [ ]
for model_class in django . apps . apps . get_models ( ) : # several validation checks to assert models will be syncing correctly
    if issubclass ( model_class , SyncableModel ) :
        name = model_class . __name__
        if _multiple_self_ref_fk_check ( model_class ) :
            raise InvalidMorangoModelConfiguration ( "Syncing models with more than 1 self referential ForeignKey is not supported." )
        try :
            from mptt import models
            from morango . utils . morango_mptt import MorangoMPTTModel , MorangoMPTTTreeManager , MorangoTreeQuerySet
            # mptt syncable model checks
            if issubclass ( model_class , models . MPTTModel ) :
                if not issubclass ( model_class , MorangoMPTTModel ) :
                    raise InvalidMorangoModelConfiguration ( "{} that inherits from MPTTModel, should instead inherit from MorangoMPTTModel." . format ( name ) )
                if not isinstance ( model_class . objects , MorangoMPTTTreeManager ) :
                    raise InvalidMPTTManager ( "Manager for {} must inherit from MorangoMPTTTreeManager." . format ( name ) )
                if not isinstance ( model_class . objects . none ( ) , MorangoTreeQuerySet ) :
                    raise InvalidMPTTQuerySet ( "Queryset for {} model must inherit from MorangoTreeQuerySet." . format ( name ) )
        except ImportError :
            pass
        # syncable model checks
        if not isinstance ( model_class . objects , SyncableModelManager ) :
            raise InvalidSyncableManager ( "Manager for {} must inherit from SyncableModelManager." . format ( name ) )
        if not isinstance ( model_class . objects . none ( ) , SyncableModelQuerySet ) :
            raise InvalidSyncableQueryset ( "Queryset for {} model must inherit from SyncableModelQuerySet." . format ( name ) )
        if model_class . _meta . many_to_many :
            raise UnsupportedFieldType ( "{} model with a ManyToManyField is not supported in morango." )
        if not hasattr ( model_class , 'morango_model_name' ) :
            raise InvalidMorangoModelConfiguration ( "{} model must define a morango_model_name attribute" . format ( name ) )
        if not hasattr ( model_class , 'morango_profile' ) :
            raise InvalidMorangoModelConfiguration ( "{} model must define a morango_profile attribute" . format ( name ) )
        # create empty list to hold model classes for profile if not yet created
        profile = model_class . morango_profile
        _profile_models [ profile ] = _profile_models . get ( profile , [ ] )
        # don ' t sync models where morango _ model _ name is None
        if model_class . morango_model_name is not None :
            _insert_model_into_profile_dict ( model_class , profile )
# for each profile , create a dict mapping from morango model names to model class
for profile , model_list in iteritems ( _profile_models ) :
    syncable_models_dict = OrderedDict ( )
    for model_class in model_list :
        syncable_models_dict [ model_class . morango_model_name ] = model_class
    _profile_models [ profile ] = syncable_models_dict


signature:
def _bulk_insert_into_app_models ( self , cursor , app_model , fields , db_values , placeholder_list ) :
    """Example query :
 ` REPLACE INTO model ( F1 , F2 , F3 ) VALUES ( % s , % s , % s ) , ( % s , % s , % s ) , ( % s , % s , % s ) `
 where values = [ 1,2,3,4,5,6,7,8,9]"""

implementation:
# calculate and create equal sized chunks of data to insert incrementally
num_of_rows_able_to_insert = self . SQLITE_MAX_VARIABLE_NUMBER // len ( fields )
num_of_values_able_to_insert = num_of_rows_able_to_insert * len ( fields )
value_chunks = [ db_values [ x : x + num_of_values_able_to_insert ] for x in range ( 0 , len ( db_values ) , num_of_values_able_to_insert ) ]
placeholder_chunks = [ placeholder_list [ x : x + num_of_rows_able_to_insert ] for x in range ( 0 , len ( placeholder_list ) , num_of_rows_able_to_insert ) ]
# insert data chunks
fields = str ( tuple ( str ( f . attname ) for f in fields ) ) . replace ( "'" , '' )
for values , params in zip ( value_chunks , placeholder_chunks ) :
    placeholder_str = ', ' . join ( params ) . replace ( "'" , '' )
    insert = """REPLACE INTO {app_model} {fields}
                        VALUES {placeholder_str}
            """ . format ( app_model = app_model , fields = fields , placeholder_str = placeholder_str )
    # use DB - APIs parameter substitution ( 2nd parameter expects a sequence )
    cursor . execute ( insert , values )


signature:
def _request ( self , endpoint , method = "GET" , lookup = None , data = { } , params = { } , userargs = None , password = None ) :
    """Generic request method designed to handle any morango endpoint .
 : param endpoint : constant representing which morango endpoint we are querying
 : param method : HTTP verb / method for request
 : param lookup : the pk value for the specific object we are querying
 : param data : dict that will be form - encoded in request
 : param params : dict to be sent as part of URL ' s query string
 : param userargs : Authorization credentials
 : param password :
 : return : ` ` Response ` ` object from request"""

implementation:
# convert user arguments into query str for passing to auth layer
if isinstance ( userargs , dict ) :
    userargs = "&" . join ( [ "{}={}" . format ( key , val ) for ( key , val ) in iteritems ( userargs ) ] )
# build up url and send request
if lookup :
    lookup = lookup + '/'
url = urljoin ( urljoin ( self . base_url , endpoint ) , lookup )
auth = ( userargs , password ) if userargs else None
resp = requests . request ( method , url , json = data , params = params , auth = auth )
resp . raise_for_status ( )
return resp


signature:
def fuzzyfinder ( input , collection , accessor = lambda x : x , sort_results = True ) :
    """Args :
 input ( str ) : A partial string which is typically entered by a user .
 collection ( iterable ) : A collection of strings which will be filtered
 based on the ` input ` .
 accessor ( function ) : If the ` collection ` is not an iterable of strings ,
 then use the accessor to fetch the string that
 will be used for fuzzy matching .
 sort _ results ( bool ) : The suggestions are sorted by considering the
 smallest contiguous match , followed by where the
 match is found in the full string . If two suggestions
 have the same rank , they are then sorted
 alpha - numerically . This parameter controls the
 * last tie - breaker - alpha - numeric sorting * . The sorting
 based on match length and position will be intact .
 Returns :
 suggestions ( generator ) : A generator object that produces a list of
 suggestions narrowed down from ` collection ` using the ` input ` ."""

implementation:
suggestions = [ ]
input = str ( input ) if not isinstance ( input , str ) else input
pat = '.*?' . join ( map ( re . escape , input ) )
pat = '(?=({0}))' . format ( pat )
# lookahead regex to manage overlapping matches
regex = re . compile ( pat , re . IGNORECASE )
for item in collection :
    r = list ( regex . finditer ( accessor ( item ) ) )
    if r :
        best = min ( r , key = lambda x : len ( x . group ( 1 ) ) )
        # find shortest match
        suggestions . append ( ( len ( best . group ( 1 ) ) , best . start ( ) , accessor ( item ) , item ) )
if sort_results :
    return ( z [ - 1 ] for z in sorted ( suggestions ) )
else :
    return ( z [ - 1 ] for z in sorted ( suggestions , key = lambda x : x [ : 2 ] ) )


signature:
def create_access_token ( self , valid_in_hours = 1 , data = None ) :
    """Creates an access token .
 TODO : check valid in hours
 TODO : maybe specify how often a token can be used"""

implementation:
data = data or { }
token = AccessToken ( token = self . generate ( ) , expires_at = expires_at ( hours = valid_in_hours ) , data = data )
return token