signature:
def get_fsapi_endpoint ( self ) :
    """Parse the fsapi endpoint from the device url ."""

implementation:
endpoint = yield from self . __session . get ( self . fsapi_device_url , timeout = self . timeout )
text = yield from endpoint . text ( encoding = 'utf-8' )
doc = objectify . fromstring ( text )
return doc . webfsapi . text


signature:
def create_session ( self ) :
    """Create a session on the frontier silicon device ."""

implementation:
req_url = '%s/%s' % ( self . __webfsapi , 'CREATE_SESSION' )
sid = yield from self . __session . get ( req_url , params = dict ( pin = self . pin ) , timeout = self . timeout )
text = yield from sid . text ( encoding = 'utf-8' )
doc = objectify . fromstring ( text )
return doc . sessionId . text


signature:
def call ( self , path , extra = None ) :
    """Execute a frontier silicon API call ."""

implementation:
try :
    if not self . __webfsapi :
        self . __webfsapi = yield from self . get_fsapi_endpoint ( )
    if not self . sid :
        self . sid = yield from self . create_session ( )
    if not isinstance ( extra , dict ) :
        extra = dict ( )
    params = dict ( pin = self . pin , sid = self . sid )
    params . update ( ** extra )
    req_url = ( '%s/%s' % ( self . __webfsapi , path ) )
    result = yield from self . __session . get ( req_url , params = params , timeout = self . timeout )
    if result . status == 200 :
        text = yield from result . text ( encoding = 'utf-8' )
    else :
        self . sid = yield from self . create_session ( )
        params = dict ( pin = self . pin , sid = self . sid )
        params . update ( ** extra )
        result = yield from self . __session . get ( req_url , params = params , timeout = self . timeout )
        text = yield from result . text ( encoding = 'utf-8' )
    return objectify . fromstring ( text )
except Exception as e :
    logging . info ( 'AFSAPI Exception: ' + traceback . format_exc ( ) )
return None


signature:
def handle_set ( self , item , value ) :
    """Helper method for setting a value by using the fsapi API ."""

implementation:
doc = yield from self . call ( 'SET/{}' . format ( item ) , dict ( value = value ) )
if doc is None :
    return None
return doc . status == 'FS_OK'


signature:
def handle_text ( self , item ) :
    """Helper method for fetching a text value ."""

implementation:
doc = yield from self . handle_get ( item )
if doc is None :
    return None
return doc . value . c8_array . text or None


signature:
def handle_int ( self , item ) :
    """Helper method for fetching a integer value ."""

implementation:
doc = yield from self . handle_get ( item )
if doc is None :
    return None
return int ( doc . value . u8 . text ) or None


signature:
def handle_long ( self , item ) :
    """Helper method for fetching a long value . Result is integer ."""

implementation:
doc = yield from self . handle_get ( item )
if doc is None :
    return None
return int ( doc . value . u32 . text ) or None


signature:
def handle_list ( self , item ) :
    """Helper method for fetching a list ( map ) value ."""

implementation:
doc = yield from self . call ( 'LIST_GET_NEXT/' + item + '/-1' , dict ( maxItems = 100 , ) )
if doc is None :
    return [ ]
if not doc . status == 'FS_OK' :
    return [ ]
ret = list ( )
for index , item in enumerate ( list ( doc . iterchildren ( 'item' ) ) ) :
    temp = dict ( band = index )
    for field in list ( item . iterchildren ( ) ) :
        temp [ field . get ( 'name' ) ] = list ( field . iterchildren ( ) ) . pop ( )
    ret . append ( temp )
return ret


signature:
def get_power ( self ) :
    """Check if the device is on ."""

implementation:
power = ( yield from self . handle_int ( self . API . get ( 'power' ) ) )
return bool ( power )


signature:
def set_power ( self , value = False ) :
    """Power on or off the device ."""

implementation:
power = ( yield from self . handle_set ( self . API . get ( 'power' ) , int ( value ) ) )
return bool ( power )


signature:
def get_modes ( self ) :
    """Get the modes supported by this device ."""

implementation:
if not self . __modes :
    self . __modes = yield from self . handle_list ( self . API . get ( 'valid_modes' ) )
return self . __modes


signature:
def get_mode_list ( self ) :
    """Get the label list of the supported modes ."""

implementation:
self . __modes = yield from self . get_modes ( )
return ( yield from self . collect_labels ( self . __modes ) )


signature:
def get_mode ( self ) :
    """Get the currently active mode on the device ( DAB , FM , Spotify ) ."""

implementation:
mode = None
int_mode = ( yield from self . handle_long ( self . API . get ( 'mode' ) ) )
modes = yield from self . get_modes ( )
for temp_mode in modes :
    if temp_mode [ 'band' ] == int_mode :
        mode = temp_mode [ 'label' ]
return str ( mode )


signature:
def set_mode ( self , value ) :
    """Set the currently active mode on the device ( DAB , FM , Spotify ) ."""

implementation:
mode = - 1
modes = yield from self . get_modes ( )
for temp_mode in modes :
    if temp_mode [ 'label' ] == value :
        mode = temp_mode [ 'band' ]
return ( yield from self . handle_set ( self . API . get ( 'mode' ) , mode ) )


signature:
def get_volume_steps ( self ) :
    """Read the maximum volume level of the device ."""

implementation:
if not self . __volume_steps :
    self . __volume_steps = yield from self . handle_int ( self . API . get ( 'volume_steps' ) )
return self . __volume_steps


signature:
def get_mute ( self ) :
    """Check if the device is muted ."""

implementation:
mute = ( yield from self . handle_int ( self . API . get ( 'mute' ) ) )
return bool ( mute )


signature:
def set_mute ( self , value = False ) :
    """Mute or unmute the device ."""

implementation:
mute = ( yield from self . handle_set ( self . API . get ( 'mute' ) , int ( value ) ) )
return bool ( mute )


signature:
def get_play_status ( self ) :
    """Get the play status of the device ."""

implementation:
status = yield from self . handle_int ( self . API . get ( 'status' ) )
return self . PLAY_STATES . get ( status )


signature:
def get_equalisers ( self ) :
    """Get the equaliser modes supported by this device ."""

implementation:
if not self . __equalisers :
    self . __equalisers = yield from self . handle_list ( self . API . get ( 'equalisers' ) )
return self . __equalisers


signature:
def get_equaliser_list ( self ) :
    """Get the label list of the supported modes ."""

implementation:
self . __equalisers = yield from self . get_equalisers ( )
return ( yield from self . collect_labels ( self . __equalisers ) )


signature:
def set_sleep ( self , value = False ) :
    """Set device sleep timer ."""

implementation: return ( yield from self . handle_set ( self . API . get ( 'sleep' ) , int ( value ) ) )


signature:
def _set_range ( self , start , stop , value , value_len ) :
    """Assumes that start and stop are already in ' buffer ' coordinates . value is a byte iterable .
 value _ len is fractional ."""

implementation:
assert stop >= start and value_len >= 0
range_len = stop - start
if range_len < value_len :
    self . _insert_zeros ( stop , stop + value_len - range_len )
    self . _copy_to_range ( start , value , value_len )
elif range_len > value_len :
    self . _del_range ( stop - ( range_len - value_len ) , stop )
    self . _copy_to_range ( start , value , value_len )
else :
    self . _copy_to_range ( start , value , value_len )


signature:
def _parse_genotype ( self , vcf_fields ) :
    """Parse genotype from VCF line data"""

implementation:
format_col = vcf_fields [ 8 ] . split ( ':' )
genome_data = vcf_fields [ 9 ] . split ( ':' )
try :
    gt_idx = format_col . index ( 'GT' )
except ValueError :
    return [ ]
return [ int ( x ) for x in re . split ( r'[\|/]' , genome_data [ gt_idx ] ) if x != '.' ]


signature:
def setDefaultIREncoding ( encoding ) :
    '''setDefaultIREncoding - Sets the default encoding used by IndexedRedis .
 This will be the default encoding used for field data . You can override this on a
 per - field basis by using an IRField ( such as IRUnicodeField or IRRawField )
 @ param encoding - An encoding ( like utf - 8)'''

implementation:
try :
    b'' . decode ( encoding )
except :
    raise ValueError ( 'setDefaultIREncoding was provided an invalid codec. Got (encoding="%s")' % ( str ( encoding ) , ) )
global defaultIREncoding
defaultIREncoding = encoding


signature:
def toIndex ( self , value ) :
    '''toIndex - An optional method which will return the value prepped for index .
 By default , " toStorage " will be called . If you provide " hashIndex = True " on the constructor ,
 the field will be md5summed for indexing purposes . This is useful for large strings , etc .'''

implementation:
if self . _isIrNull ( value ) :
    ret = IR_NULL_STR
else :
    ret = self . _toIndex ( value )
if self . isIndexHashed is False :
    return ret
return md5 ( tobytes ( ret ) ) . hexdigest ( )


signature:
def _getReprProperties ( self ) :
    '''_ getReprProperties - Get the properties of this field to display in repr ( ) .
 These should be in the form of $ propertyName = $ propertyRepr
 The default IRField implementation handles just the " hashIndex " property .
 defaultValue is part of " _ _ repr _ _ " impl . You should just extend this method
 with your object ' s properties instead of rewriting repr .'''

implementation:
ret = [ ]
if getattr ( self , 'valueType' , None ) is not None :
    ret . append ( 'valueType=%s' % ( self . valueType . __name__ , ) )
if hasattr ( self , 'hashIndex' ) :
    ret . append ( 'hashIndex=%s' % ( self . hashIndex , ) )
return ret


signature:
def copy ( self ) :
    '''copy - Create a copy of this IRField .
 Each subclass should implement this , as you ' ll need to pass in the args to constructor .
 @ return < IRField ( or subclass ) > - Another IRField that has all the same values as this one .'''

implementation: return self . __class__ ( name = self . name , valueType = self . valueType , defaultValue = self . defaultValue , hashIndex = self . hashIndex )


signature:
def getObj ( self ) :
    '''getObj - Fetch ( if not fetched ) and return the obj associated with this data .'''

implementation:
if self . obj is None :
    if not self . pk :
        return None
    self . obj = self . foreignModel . objects . get ( self . pk )
return self . obj


signature:
def getPk ( self ) :
    '''getPk - Resolve any absent pk ' s off the obj ' s ( like if an obj has been saved ) , and return the pk .'''

implementation:
if not self . pk and self . obj :
    if self . obj . _id :
        self . pk = self . obj . _id
return self . pk


signature:
def objHasUnsavedChanges ( self ) :
    '''objHasUnsavedChanges - Check if any object has unsaved changes , cascading .'''

implementation:
if not self . obj :
    return False
return self . obj . hasUnsavedChanges ( cascadeObjects = True )


signature:
def getPk ( self ) :
    '''getPk - @ see ForeignLinkData . getPk'''

implementation:
if not self . pk or None in self . pk :
    for i in range ( len ( self . pk ) ) :
        if self . pk [ i ] :
            continue
        if self . obj [ i ] and self . obj [ i ] . _id :
            self . pk [ i ] = self . obj [ i ] . _id
return self . pk


signature:
def getObj ( self ) :
    '''getObj - @ see ForeignLinkData . getObj
 Except this always returns a list'''

implementation:
if self . obj :
    needPks = [ ( i , self . pk [ i ] ) for i in range ( len ( self . obj ) ) if self . obj [ i ] is None ]
    if not needPks :
        return self . obj
    fetched = list ( self . foreignModel . objects . getMultiple ( [ needPk [ 1 ] for needPk in needPks ] ) )
    i = 0
    for objIdx , pk in needPks :
        self . obj [ objIdx ] = fetched [ i ]
        i += 1
return self . obj


signature:
def isFetched ( self ) :
    '''isFetched - @ see ForeignLinkData . isFetched'''

implementation:
if not self . obj :
    return False
if not self . pk or None in self . obj :
    return False
return not bool ( self . obj is None )


signature:
def objHasUnsavedChanges ( self ) :
    '''objHasUnsavedChanges - @ see ForeignLinkData . objHasUnsavedChanges
 True if ANY object has unsaved changes .'''

implementation:
if not self . obj :
    return False
for thisObj in self . obj :
    if not thisObj :
        continue
    if thisObj . hasUnsavedChanges ( cascadeObjects = True ) :
        return True
return False


signature:
def assert_json_type ( value : JsonValue , expected_type : JsonCheckType ) -> None :
    """Check that a value has a certain JSON type .
 Raise TypeError if the type does not match .
 Supported types : str , int , float , bool , list , dict , and None .
 float will match any number , int will only match numbers without
 fractional part .
 The special type JList ( x ) will match a list value where each
 item is of type x :
 > > > assert _ json _ type ( [ 1 , 2 , 3 ] , JList ( int ) )"""

implementation:
def type_name ( t : Union [ JsonCheckType , Type [ None ] ] ) -> str :
    if t is None :
        return "None"
    if isinstance ( t , JList ) :
        return "list"
    return t . __name__
if expected_type is None :
    if value is None :
        return
elif expected_type == float :
    if isinstance ( value , float ) or isinstance ( value , int ) :
        return
elif expected_type in [ str , int , bool , list , dict ] :
    if isinstance ( value , expected_type ) : # type : ignore
        return
elif isinstance ( expected_type , JList ) :
    if isinstance ( value , list ) :
        for v in value :
            assert_json_type ( v , expected_type . value_type )
        return
else :
    raise TypeError ( "unsupported type" )
raise TypeError ( "wrong JSON type {} != {}" . format ( type_name ( expected_type ) , type_name ( type ( value ) ) ) )


signature:
def json_get ( json : JsonValue , path : str , expected_type : Any = ANY ) -> Any :
    """Get a JSON value by path , optionally checking its type .
 > > > j = { " foo " : { " num " : 3.4 , " s " : " Text " } , " arr " : [ 10 , 20 , 30 ] }
 > > > json _ get ( j , " / foo / num " )
 3.4
 > > > json _ get ( j , " / arr [ 1 ] " )
 20
 Raise ValueError if the path is not found :
 > > > json _ get ( j , " / foo / unknown " )
 Traceback ( most recent call last ) :
 ValueError : JSON path ' / foo / unknown ' not found
 Raise TypeError if the path contains a non - object element :
 > > > json _ get ( j , " / foo / num / bar " )
 Traceback ( most recent call last ) :
 TypeError : JSON path ' / foo / num ' is not an object
 Or a non - array element :
 > > > json _ get ( j , " / foo [ 2 ] " )
 Traceback ( most recent call last ) :
 TypeError : JSON path ' / foo ' is not an array
 Raise an IndexError if the array index is out of bounds :
 > > > json _ get ( j , " / arr [ 10 ] " )
 Traceback ( most recent call last ) :
 IndexError : JSON array ' / arr ' too small ( 3 < = 10)
 Recognized types are : str , int , float , bool , list , dict , and None .
 TypeError is raised if the type does not match .
 > > > json _ get ( j , " / foo / num " , str )
 Traceback ( most recent call last ) :
 TypeError : wrong JSON type str ! = float
 float will match any number , int will only match numbers without a
 fractional part .
 > > > json _ get ( j , " / foo / num " , float )
 3.4
 > > > json _ get ( j , " / foo / num " , int )
 Traceback ( most recent call last ) :
 TypeError : wrong JSON type int ! = float"""

implementation:
elements = _parse_json_path ( path )
current = json
current_path = ""
for i , element in enumerate ( elements ) :
    if isinstance ( element , str ) :
        if not isinstance ( current , dict ) :
            msg = "JSON path '{}' is not an object" . format ( current_path )
            raise TypeError ( msg ) from None
        if element not in current :
            raise ValueError ( "JSON path '{}' not found" . format ( path ) )
        current_path += "/" + element
        current = current [ element ]
    else :
        if not isinstance ( current , list ) :
            msg = "JSON path '{}' is not an array" . format ( current_path )
            raise TypeError ( msg ) from None
        if element >= len ( current ) :
            msg = "JSON array '{}' too small ({} <= {})" . format ( current_path , len ( current ) , element )
            raise IndexError ( msg )
        current_path += "[{}]" . format ( i )
        current = current [ element ]
if expected_type != ANY :
    assert_json_type ( current , cast ( JsonType , expected_type ) )
return current


signature:
def json_get_default ( json : JsonValue , path : str , default : Any , expected_type : Any = ANY ) -> Any :
    """Get a JSON value by path , optionally checking its type .
 This works exactly like json _ get ( ) , but instead of raising
 ValueError or IndexError when a path part is not found , return
 the provided default value :
 > > > json _ get _ default ( { } , " / foo " , " I am a default value " )
 ' I am a default value '
 TypeErrors will be raised as in json _ get ( ) if an expected _ type
 is provided :
 > > > json _ get _ default ( { " foo " : " bar " } , " / foo " , 123 , int )
 Traceback ( most recent call last ) :
 TypeError : wrong JSON type int ! = str"""

implementation:
try :
    return json_get ( json , path , expected_type )
except ( ValueError , IndexError ) :
    return default


signature:
def load ( cls , fh ) :
    """Load json or yaml data from file handle .
 Args :
 fh ( file ) : File handle to load from .
 Examlple :
 > > > with open ( ' data . json ' , ' r ' ) as json :
 > > > jsdata = composite . load ( json )
 > > > with open ( ' data . yml ' , ' r ' ) as yml :
 > > > ymldata = composite . load ( yml )"""

implementation:
dat = fh . read ( )
try :
    ret = cls . from_json ( dat )
except :
    ret = cls . from_yaml ( dat )
return ret


signature:
def from_json ( cls , fh ) :
    """Load json from file handle .
 Args :
 fh ( file ) : File handle to load from .
 Examlple :
 > > > with open ( ' data . json ' , ' r ' ) as json :
 > > > data = composite . load ( json )"""

implementation:
if isinstance ( fh , str ) :
    return cls ( json . loads ( fh ) )
else :
    return cls ( json . load ( fh ) )


signature:
def intersection ( self , other , recursive = True ) :
    """Recursively compute intersection of data . For dictionaries , items
 for specific keys will be reduced to unique items . For lists , items
 will be reduced to unique items . This method is meant to be analogous
 to set . intersection for composite objects .
 Args :
 other ( composite ) : Other composite object to intersect with .
 recursive ( bool ) : Whether or not to perform the operation recursively ,
 for all nested composite objects ."""

implementation:
if not isinstance ( other , composite ) :
    raise AssertionError ( 'Cannot intersect composite and {} types' . format ( type ( other ) ) )
if self . meta_type != other . meta_type :
    return composite ( { } )
if self . meta_type == 'list' :
    keep = [ ]
    for item in self . _list :
        if item in other . _list :
            if recursive and isinstance ( item , composite ) :
                keep . extend ( item . intersection ( other . index ( item ) , recursive = True ) )
            else :
                keep . append ( item )
    return composite ( keep )
elif self . meta_type == 'dict' :
    keep = { }
    for key in self . _dict :
        item = self . _dict [ key ]
        if key in other . _dict :
            if recursive and isinstance ( item , composite ) and isinstance ( other . get ( key ) , composite ) :
                keep [ key ] = item . intersection ( other . get ( key ) , recursive = True )
            elif item == other [ key ] :
                keep [ key ] = item
    return composite ( keep )
return


signature:
def union ( self , other , recursive = True , overwrite = False ) :
    """Recursively compute union of data . For dictionaries , items
 for specific keys will be combined into a list , depending on the
 status of the overwrite = parameter . For lists , items will be appended
 and reduced to unique items . This method is meant to be analogous
 to set . union for composite objects .
 Args :
 other ( composite ) : Other composite object to union with .
 recursive ( bool ) : Whether or not to perform the operation recursively ,
 for all nested composite objects .
 overwrite ( bool ) : Whether or not to overwrite entries with the same
 key in a nested dictionary ."""

implementation:
if not isinstance ( other , composite ) :
    raise AssertionError ( 'Cannot union composite and {} types' . format ( type ( other ) ) )
if self . meta_type != other . meta_type :
    return composite ( [ self , other ] )
if self . meta_type == 'list' :
    keep = [ ]
    for item in self . _list :
        keep . append ( item )
    for item in other . _list :
        if item not in self . _list :
            keep . append ( item )
    return composite ( keep )
elif self . meta_type == 'dict' :
    keep = { }
    for key in list ( set ( list ( self . _dict . keys ( ) ) + list ( other . _dict . keys ( ) ) ) ) :
        left = self . _dict . get ( key )
        right = other . _dict . get ( key )
        if recursive and isinstance ( left , composite ) and isinstance ( right , composite ) :
            keep [ key ] = left . union ( right , recursive = recursive , overwrite = overwrite )
        elif left == right :
            keep [ key ] = left
        elif left is None :
            keep [ key ] = right
        elif right is None :
            keep [ key ] = left
        elif overwrite :
            keep [ key ] = right
        else :
            keep [ key ] = composite ( [ left , right ] )
    return composite ( keep )
return


signature:
def update ( self , other ) :
    """Update internal dictionary object . This is meant to be an
 analog for dict . update ( ) ."""

implementation:
if self . meta_type == 'list' :
    raise AssertionError ( 'Cannot update object of `list` base type!' )
elif self . meta_type == 'dict' :
    self . _dict = dict ( self + composite ( other ) )
    return


signature:
def items ( self ) :
    """Return keys for object , if they are available ."""

implementation:
if self . meta_type == 'list' :
    return self . _list
elif self . meta_type == 'dict' :
    return self . _dict . items ( )


signature:
def values ( self ) :
    """Return keys for object , if they are available ."""

implementation:
if self . meta_type == 'list' :
    return self . _list
elif self . meta_type == 'dict' :
    return self . _dict . values ( )


signature:
def append ( self , item ) :
    """Append to object , if object is list ."""

implementation:
if self . meta_type == 'dict' :
    raise AssertionError ( 'Cannot append to object of `dict` base type!' )
if self . meta_type == 'list' :
    self . _list . append ( item )
return


signature:
def extend ( self , item ) :
    """Extend list from object , if object is list ."""

implementation:
if self . meta_type == 'dict' :
    raise AssertionError ( 'Cannot extend to object of `dict` base type!' )
if self . meta_type == 'list' :
    self . _list . extend ( item )
return


signature:
def json ( self ) :
    """Return JSON representation of object ."""

implementation:
if self . meta_type == 'list' :
    ret = [ ]
    for dat in self . _list :
        if not isinstance ( dat , composite ) :
            ret . append ( dat )
        else :
            ret . append ( dat . json ( ) )
    return ret
elif self . meta_type == 'dict' :
    ret = { }
    for key in self . _dict :
        if not isinstance ( self . _dict [ key ] , composite ) :
            ret [ key ] = self . _dict [ key ]
        else :
            ret [ key ] = self . _dict [ key ] . json ( )
    return ret


signature:
def write_json ( self , fh , pretty = True ) :
    """Write composite object to file handle in JSON format .
 Args :
 fh ( file ) : File handle to write to .
 pretty ( bool ) : Sort keys and indent in output ."""

implementation:
sjson = json . JSONEncoder ( ) . encode ( self . json ( ) )
if pretty :
    json . dump ( json . loads ( sjson ) , fh , sort_keys = True , indent = 4 )
else :
    json . dump ( json . loads ( sjson ) , fh )
return


signature:
def write ( self , fh , pretty = True ) :
    """API niceness defaulting to composite . write _ json ( ) ."""

implementation: return self . write_json ( fh , pretty = pretty )


implementation:
data = { }
for item in self . _data :
    if isinstance ( self . _data [ item ] , filetree ) :
        data [ item ] = self . _data [ item ] . json ( )
    else :
        data [ item ] = self . _data [ item ]
return data


signature:
def filelist ( self ) :
    """Return list of files in filetree ."""

implementation:
if len ( self . _filelist ) == 0 :
    for item in self . _data :
        if isinstance ( self . _data [ item ] , filetree ) :
            self . _filelist . extend ( self . _data [ item ] . filelist ( ) )
        else :
            self . _filelist . append ( self . _data [ item ] )
return self . _filelist


signature:
def prune ( self , regex = r".*" ) :
    """Prune leaves of filetree according to specified
 regular expression .
 Args :
 regex ( str ) : Regular expression to use in pruning tree ."""

implementation: return filetree ( self . root , ignore = self . ignore , regex = regex )


signature:
def deref ( self , ctx ) :
    """Returns the value this reference is pointing to . This method uses ' ctx ' to resolve the reference and return
 the value this reference references .
 If the call was already made , it returns a cached result .
 It also makes sure there ' s no cyclic reference , and if so raises CyclicReferenceError ."""

implementation:
if self in ctx . call_nodes :
    raise CyclicReferenceError ( ctx , self )
if self in ctx . cached_results :
    return ctx . cached_results [ self ]
try :
    ctx . call_nodes . add ( self )
    ctx . call_stack . append ( self )
    result = self . evaluate ( ctx )
    ctx . cached_results [ self ] = result
    return result
except :
    if ctx . exception_call_stack is None :
        ctx . exception_call_stack = list ( ctx . call_stack )
    raise
finally :
    ctx . call_stack . pop ( )
    ctx . call_nodes . remove ( self )


signature:
def getModel ( self ) :
    '''getModel - get the IndexedRedisModel associated with this list . If one was not provided in constructor ,
 it will be inferred from the first item in the list ( if present )
 @ return < None / IndexedRedisModel > - None if none could be found , otherwise the IndexedRedisModel type of the items in this list .
 @ raises ValueError if first item is not the expected type .'''

implementation:
if not self . mdl and len ( self ) > 0 :
    mdl = self [ 0 ] . __class__
    self . __validate_model ( mdl )
    self . mdl = mdl
return self . mdl


signature:
def delete ( self ) :
    '''delete - Delete all objects in this list .
 @ return < int > - Number of objects deleted'''

implementation:
if len ( self ) == 0 :
    return 0
mdl = self . getModel ( )
return mdl . deleter . deleteMultiple ( self )


signature:
def save ( self ) :
    '''save - Save all objects in this list'''

implementation:
if len ( self ) == 0 :
    return [ ]
mdl = self . getModel ( )
return mdl . saver . save ( self )


signature:
def reload ( self ) :
    '''reload - Reload all objects in this list .
 Updates in - place . To just fetch all these objects again , use " refetch "
 @ return - List ( same order as current objects ) of either exception ( KeyError ) if operation failed ,
 or a dict of fields changed - > ( old , new )'''

implementation:
if len ( self ) == 0 :
    return [ ]
ret = [ ]
for obj in self :
    res = None
    try :
        res = obj . reload ( )
    except Exception as e :
        res = e
    ret . append ( res )
return ret


signature:
def refetch ( self ) :
    '''refetch - Fetch a fresh copy of all items in this list .
 Returns a new list . To update in - place , use " reload " .
 @ return IRQueryableList < IndexedRedisModel > - List of fetched items'''

implementation:
if len ( self ) == 0 :
    return IRQueryableList ( )
mdl = self . getModel ( )
pks = [ item . _id for item in self if item . _id ]
return mdl . objects . getMultiple ( pks )


signature:
def hashDictOneLevel ( myDict ) :
    '''A function which can generate a hash of a one - level
 dict containing strings ( like REDIS _ CONNECTION _ PARAMS )
 @ param myDict < dict > - Dict with string keys and values
 @ return < long > - Hash of myDict'''

implementation:
keys = [ str ( x ) for x in myDict . keys ( ) ]
keys . sort ( )
lst = [ ]
for key in keys :
    lst . append ( str ( myDict [ key ] ) + '__~~__' )
return '+_[,' . join ( lst ) . __hash__ ( )


signature:
def output ( self , to = None , * args , ** kwargs ) :
    '''Outputs to a stream ( like a file or request )'''

implementation:
for blok in self :
    blok . output ( to , * args , ** kwargs )
return self


signature:
def render ( self , * args , ** kwargs ) :
    '''Renders as a str'''

implementation:
render_to = StringIO ( )
self . output ( render_to , * args , ** kwargs )
return render_to . getvalue ( )


signature:
def output ( self , to = None , formatted = False , indent = 0 , indentation = '  ' , * args , ** kwargs ) :
    '''Outputs to a stream ( like a file or request )'''

implementation:
if formatted and self . blox :
    self . blox [ 0 ] . output ( to = to , formatted = True , indent = indent , indentation = indentation , * args , ** kwargs )
    for blok in self . blox [ 1 : ] :
        to . write ( '\n' )
        to . write ( indent * indentation )
        blok . output ( to = to , formatted = True , indent = indent , indentation = indentation , * args , ** kwargs )
    if not indent :
        to . write ( '\n' )
else :
    for blok in self . blox :
        blok . output ( to = to , * args , ** kwargs )
return self


signature:
def start_tag ( self ) :
    '''Returns the elements HTML start tag'''

implementation:
direct_attributes = ( attribute . render ( self ) for attribute in self . render_attributes )
attributes = ( )
if hasattr ( self , '_attributes' ) :
    attributes = ( '{0}="{1}"' . format ( key , value ) for key , value in self . attributes . items ( ) if value )
rendered_attributes = " " . join ( filter ( bool , chain ( direct_attributes , attributes ) ) )
return '<{0}{1}{2}{3}>' . format ( self . tag , ' ' if rendered_attributes else '' , rendered_attributes , ' /' if self . tag_self_closes else "" )


implementation:
to . write ( self . start_tag )
if not self . tag_self_closes :
    to . write ( self . end_tag )


implementation:
if formatted :
    to . write ( self . start_tag )
    to . write ( '\n' )
    if not self . tag_self_closes :
        for blok in self . blox :
            to . write ( indentation * ( indent + 1 ) )
            blok . output ( to = to , indent = indent + 1 , formatted = True , indentation = indentation , * args , ** kwargs )
            to . write ( '\n' )
    to . write ( indentation * indent )
    to . write ( self . end_tag )
    if not indentation :
        to . write ( '\n' )
else :
    to . write ( self . start_tag )
    if not self . tag_self_closes :
        for blok in self . blox :
            blok . output ( to = to , * args , ** kwargs )
    to . write ( self . end_tag )


signature:
def safe_repr ( obj ) :
    """Returns a repr of an object and falls back to a minimal representation of type and ID if the call to repr raised
 an error .
 : param obj : object to safe repr
 : returns : repr string or ' ( type < id > repr error ) ' string
 : rtype : str"""

implementation:
try :
    obj_repr = repr ( obj )
except :
    obj_repr = "({0}<{1}> repr error)" . format ( type ( obj ) , id ( obj ) )
return obj_repr


signature:
def output ( self , to = None , formatted = False , * args , ** kwargs ) :
    '''Outputs the set text'''

implementation: to . write ( '<!DOCTYPE {0}>' . format ( self . type ) )


signature:
def match_to_clinvar ( genome_file , clin_file ) :
    """Match a genome VCF to variants in the ClinVar VCF file
 Acts as a generator , yielding tuples of :
 ( ClinVarVCFLine , ClinVarAllele , zygosity )
 ' zygosity ' is a string and corresponds to the genome ' s zygosity for that
 ClinVarAllele . It can be either : ' Het ' ( heterozygous ) , ' Hom ' ( homozygous ) ,
 or ' Hem ' ( hemizygous , e . g . X chromosome in XY individuals ) ."""

implementation:
clin_curr_line = _next_line ( clin_file )
genome_curr_line = _next_line ( genome_file )
# Ignores all the lines that start with a hashtag
while clin_curr_line . startswith ( '#' ) :
    clin_curr_line = _next_line ( clin_file )
while genome_curr_line . startswith ( '#' ) :
    genome_curr_line = _next_line ( genome_file )
# Advance through both files simultaneously to find matches
while clin_curr_line and genome_curr_line : # Advance a file when positions aren ' t equal .
    clin_curr_pos = VCFLine . get_pos ( clin_curr_line )
    genome_curr_pos = VCFLine . get_pos ( genome_curr_line )
    try :
        if clin_curr_pos [ 'chrom' ] > genome_curr_pos [ 'chrom' ] :
            genome_curr_line = _next_line ( genome_file )
            continue
        elif clin_curr_pos [ 'chrom' ] < genome_curr_pos [ 'chrom' ] :
            clin_curr_line = _next_line ( clin_file )
            continue
        if clin_curr_pos [ 'pos' ] > genome_curr_pos [ 'pos' ] :
            genome_curr_line = _next_line ( genome_file )
            continue
        elif clin_curr_pos [ 'pos' ] < genome_curr_pos [ 'pos' ] :
            clin_curr_line = _next_line ( clin_file )
            continue
    except StopIteration :
        break
    # If we get here , start positions match .
    # Look for allele matching .
    genome_vcf_line = GenomeVCFLine ( vcf_line = genome_curr_line , skip_info = True )
    # We can skip if genome has no allele information for this point .
    if not genome_vcf_line . genotype_allele_indexes :
        genome_curr_line = _next_line ( genome_file )
        continue
    # Match only if ClinVar and Genome ref _ alleles match .
    clinvar_vcf_line = ClinVarVCFLine ( vcf_line = clin_curr_line )
    if not genome_vcf_line . ref_allele == clinvar_vcf_line . ref_allele :
        try :
            genome_curr_line = _next_line ( genome_file )
            clin_curr_line = _next_line ( clin_file )
            continue
        except StopIteration :
            break
    # Determine genome alleles and zygosity . Zygosity is assumed to be one
    # of : heterozygous , homozygous , or hemizygous .
    genotype_allele_indexes = genome_vcf_line . genotype_allele_indexes
    genome_alleles = [ genome_vcf_line . alleles [ x ] for x in genotype_allele_indexes ]
    if len ( genome_alleles ) == 1 :
        zygosity = 'Hem'
    elif len ( genome_alleles ) == 2 :
        if genome_alleles [ 0 ] . sequence == genome_alleles [ 1 ] . sequence :
            zygosity = 'Hom'
            genome_alleles = [ genome_alleles [ 0 ] ]
        else :
            zygosity = 'Het'
    else :
        raise ValueError ( 'This code only expects to work on genomes ' + 'with one or two alleles called at each ' + 'location. The following line violates this:' + str ( genome_vcf_line ) )
    # Look for matches to ClinVar alleles .
    for genome_allele in genome_alleles :
        for allele in clinvar_vcf_line . alleles :
            if genome_allele . sequence == allele . sequence : # The ' records ' attribute is specific to ClinVarAlleles .
                if hasattr ( allele , 'records' ) :
                    yield ( genome_vcf_line , allele , zygosity )
    # Done matching , move on .
    try :
        genome_curr_line = _next_line ( genome_file )
        clin_curr_line = _next_line ( clin_file )
    except StopIteration :
        break


signature:
def as_dict ( self ) :
    """Return Allele data as dict object ."""

implementation:
self_as_dict = dict ( )
self_as_dict [ 'sequence' ] = self . sequence
if hasattr ( self , 'frequency' ) :
    self_as_dict [ 'frequency' ] = self . frequency
return self_as_dict


signature:
def _parse_allele_data ( self ) :
    """Create list of Alleles from VCF line data"""

implementation: return [ Allele ( sequence = x ) for x in [ self . ref_allele ] + self . alt_alleles ]


signature:
def _parse_info ( self , info_field ) :
    """Parse the VCF info field"""

implementation:
info = dict ( )
for item in info_field . split ( ';' ) : # Info fields may be " foo = bar " or just " foo " .
# For the first case , store key " foo " with value " bar "
# For the second case , store key " foo " with value True .
    info_item_data = item . split ( '=' )
    # If length is one , just store as a key with value = true .
    if len ( info_item_data ) == 1 :
        info [ info_item_data [ 0 ] ] = True
    elif len ( info_item_data ) == 2 :
        info [ info_item_data [ 0 ] ] = info_item_data [ 1 ]
return info


signature:
def as_dict ( self ) :
    """Dict representation of parsed VCF data"""

implementation:
self_as_dict = { 'chrom' : self . chrom , 'start' : self . start , 'ref_allele' : self . ref_allele , 'alt_alleles' : self . alt_alleles , 'alleles' : [ x . as_dict ( ) for x in self . alleles ] }
try :
    self_as_dict [ 'info' ] = self . info
except AttributeError :
    pass
return self_as_dict


signature:
def get_pos ( vcf_line ) :
    """Very lightweight parsing of a vcf line to get position .
 Returns a dict containing :
 ' chrom ' : index of chromosome ( int ) , indicates sort order
 ' pos ' : position on chromosome ( int )"""

implementation:
if not vcf_line :
    return None
vcf_data = vcf_line . strip ( ) . split ( '\t' )
return_data = dict ( )
return_data [ 'chrom' ] = CHROM_INDEX [ vcf_data [ 0 ] ]
return_data [ 'pos' ] = int ( vcf_data [ 1 ] )
return return_data


signature:
def _toStorage ( self , value ) :
    '''_ toStorage - Convert the value to a string representation for storage .
 @ param value - The value of the item to convert
 @ return A string value suitable for storing .'''

implementation:
for chainedField in self . chainedFields :
    value = chainedField . toStorage ( value )
return value


signature:
def _fromStorage ( self , value ) :
    '''_ fromStorage - Convert the value from storage ( string ) to the value type .
 @ return - The converted value , or " irNull " if no value was defined ( and field type is not default / string )'''

implementation:
for chainedField in reversed ( self . chainedFields ) :
    value = chainedField . _fromStorage ( value )
return value


signature:
def _checkCanIndex ( self ) :
    '''_ checkCanIndex - Check if we CAN index ( if all fields are indexable ) .
 Also checks the right - most field for " hashIndex " - if it needs to hash we will hash .'''

implementation:
# NOTE : We can ' t just check the right - most field . For types like pickle that don ' t support indexing , they don ' t
# support it because python2 and python3 have different results for pickle . dumps on the same object .
# So if we have a field chain like Pickle , Compressed then we will have two different results .
if not self . chainedFields :
    return ( False , False )
for chainedField in self . chainedFields :
    if chainedField . CAN_INDEX is False :
        return ( False , False )
return ( True , self . chainedFields [ - 1 ] . hashIndex )


signature:
def byte_length ( self ) :
    """: returns : sum of lengthes of all ranges or None if one of the ranges is open
 : rtype : int , float or None"""

implementation:
sum = 0
for r in self :
    if r . is_open ( ) :
        return None
    sum += r . byte_length ( )
return sum


signature:
def has_overlaps ( self ) :
    """: returns : True if one or more range in the list overlaps with another
 : rtype : bool"""

implementation:
sorted_list = sorted ( self )
for i in range ( 0 , len ( sorted_list ) - 1 ) :
    if sorted_list [ i ] . overlaps ( sorted_list [ i + 1 ] ) :
        return True
return False


signature:
def max_stop ( self ) :
    """: returns : maximum stop in list or None if there ' s at least one open range
 : type : int , float or None"""

implementation:
m = 0
for r in self :
    if r . is_open ( ) :
        return None
    m = max ( m , r . stop )
return m


signature:
def byte_offset ( self , bytes ) :
    """Maps ` bytes ` length to a sequence ' s offset . For example , if we do byte _ offset ( 5 ) and our list of sequences is
 [ ( 0 , 2 ) , ( 10 , 11 ) , ( 40 , 45 ) ] then the returned value will be 42.
 Note that ` bytes ` must be < = byte _ length ( ) .
 : returns : actual offset in one of the sequences in the range for request byte length .
 : rtype : int or float"""

implementation:
remaining_bytes = bytes
for r in self :
    if r . is_open ( ) or r . byte_length ( ) >= remaining_bytes :
        return r . start + remaining_bytes
    else :
        remaining_bytes -= r . byte_length ( )
assert False , "requested byte offset {0!r} is outside the range list {1!r}" . format ( bytes , self )


signature:
def main ( ) :
    """Parse command line argument and
 output appropriate file type ( csv or JSON )"""

implementation:
parser = ArgumentParser ( )
parser . add_argument ( "-c" , "--clinvarfile" , dest = "clinvarfile" , help = "ClinVar VCF file (either this or -C must be specified)" , metavar = "CLINVARFILE" )
parser . add_argument ( "-C" , "--clinvardir" , dest = "clinvardir" , help = "ClinVar VCF directory (either this or -c must be specified). " + "This option will use vcf2clinvar.clinvar_update to automatically " + "check and import the most recent ClinVar file to this directory." , metavar = "CLINVARDIR" )
parser . add_argument ( "-i" , "--input" , dest = "inputfile" , help = "Input VCF file ['.vcf', '.vcf.gz', '.vcf.bz2']. " + "Uncompressed genome data is also accepted via stdin." , metavar = "INPUT" )
parser . add_argument ( "-t" , "--type" , dest = "type" , default = 'csv' , help = "Output report type ('csv' or 'json'). Defaults to csv. " + "CSV Report: Reports all genome variants matching ClinVar records, " + "and some summary ClinVar data from these records. Header lines " + "with metadata begin with '##'.\n" + "JSON Report: Reports genome variants matching ClinVar records " + "(no record information is included)." , metavar = "TYPE" )
parser . add_argument ( "-n" , "--notes" , dest = "notes" , help = "Notes (JSON format) to include in report. (JSON report only)" , metavar = "NOTES" )
parser . add_argument ( "-g" , "--genome-build" , dest = "build" , help = "Genome build to include in report ('b37' or 'b38')." , metavar = "GENOMEBUILD" )
options = parser . parse_args ( )
version = os . popen ( "python setup.py --version" ) . read ( ) . strip ( )
if options . inputfile :
    if options . inputfile . endswith ( '.vcf' ) :
        input_genome_file = open ( options . inputfile )
    elif options . inputfile . endswith ( '.vcf.gz' ) :
        input_genome_file = gzip . open ( options . inputfile )
    elif options . inputfile . endswith ( '.vcf.bz2' ) :
        input_genome_file = bz2 . BZ2File ( options . inputfile )
    else :
        raise IOError ( "Genome filename expected to end with ''.vcf'," + " '.vcf.gz', or '.vcf.bz2'." )
elif not sys . stdin . isatty ( ) :
    input_genome_file = sys . stdin
else :
    sys . stderr . write ( "Provide input VCF file\n" )
    parser . print_help ( )
    sys . exit ( 1 )
if options . build and options . build in [ 'b37' , 'b38' ] :
    build = options . build
else :
    raise IOError ( "Input VCF genome build must be 'b37' or 'b38'." )
if ( not ( options . clinvarfile or options . clinvardir ) or ( options . clinvarfile and options . clinvardir ) ) :
    sys . stderr . write ( "Please provide either a ClinVar file or directory." )
    parser . print_help ( )
    sys . exit ( 1 )
if options . clinvarfile :
    clinvarfilename = options . clinvarfile
elif options . clinvardir :
    clinvarfilename = get_latest_vcf_file ( target_dir = options . clinvardir , build = build )
if clinvarfilename . endswith ( '.vcf' ) :
    input_clinvar_file = open ( options . clinvarfile )
elif clinvarfilename . endswith ( '.vcf.gz' ) :
    input_clinvar_file = gzip . open ( clinvarfilename )
elif clinvarfilename . endswith ( '.vcf.bz2' ) :
    input_clinvar_file = bz2 . BZ2File ( clinvarfilename )
else :
    raise IOError ( "ClinVar filename expected to end with '.vcf'," + " '.vcf.gz', or '.vcf.bz2'." )
if options . type not in [ 'csv' , 'json' ] :
    raise IOError ( "Not a valid report type, must be 'csv' or 'json'." )
if options . type == "csv" :
    csv_report ( input_genome_file = input_genome_file , input_clinvar_file = input_clinvar_file , build = build , version = version )
elif options . type == "json" :
    notes_json = { }
    if options . notes :
        notes_json [ "parameter" ] = options . notes
        try :
            notes_json = json . loads ( options . notes )
        except :
            sys . stderr . write ( "Could not parse JSON notes field\n" )
    json_report ( input_genome_file = input_genome_file , input_clinvar_file = input_clinvar_file , build = build , notes = notes_json , version = version )


signature:
def nav_to_vcf_dir ( ftp , build ) :
    """Navigate an open ftplib . FTP to appropriate directory for ClinVar VCF files .
 Args :
 ftp : ( type : ftplib . FTP ) an open connection to ftp . ncbi . nlm . nih . gov
 build : ( type : string ) genome build , either ' b37 ' or ' b38'"""

implementation:
if build == 'b37' :
    ftp . cwd ( DIR_CLINVAR_VCF_B37 )
elif build == 'b38' :
    ftp . cwd ( DIR_CLINVAR_VCF_B38 )
else :
    raise IOError ( "Genome build not recognized." )


signature:
def as_dict ( self , * args , ** kwargs ) :
    """Return ClinVarAllele data as dict object ."""

implementation:
self_as_dict = super ( ClinVarAllele , self ) . as_dict ( * args , ** kwargs )
self_as_dict [ 'hgvs' ] = self . hgvs
self_as_dict [ 'clnalleleid' ] = self . clnalleleid
self_as_dict [ 'clnsig' ] = self . clnsig
self_as_dict [ 'clndn' ] = self . clndn
self_as_dict [ 'clndisdb' ] = self . clndisdb
self_as_dict [ 'clnvi' ] = self . clnvi
return self_as_dict


signature:
def _parse_frequencies ( self ) :
    """Parse frequency data in ClinVar VCF"""

implementation:
frequencies = OrderedDict ( [ ( 'EXAC' , 'Unknown' ) , ( 'ESP' , 'Unknown' ) , ( 'TGP' , 'Unknown' ) ] )
pref_freq = 'Unknown'
for source in frequencies . keys ( ) :
    freq_key = 'AF_' + source
    if freq_key in self . info :
        frequencies [ source ] = self . info [ freq_key ]
        if pref_freq == 'Unknown' :
            pref_freq = frequencies [ source ]
return pref_freq , frequencies


signature:
def _parse_allele_data ( self ) :
    """Parse alleles for ClinVar VCF , overrides parent method ."""

implementation:
# Get allele frequencies if they exist .
pref_freq , frequencies = self . _parse_frequencies ( )
info_clnvar_single_tags = [ 'ALLELEID' , 'CLNSIG' , 'CLNHGVS' ]
cln_data = { x . lower ( ) : self . info [ x ] if x in self . info else None for x in info_clnvar_single_tags }
cln_data . update ( { 'clndisdb' : [ x . split ( ',' ) for x in self . info [ 'CLNDISDB' ] . split ( '|' ) ] if 'CLNDISDB' in self . info else [ ] } )
cln_data . update ( { 'clndn' : self . info [ 'CLNDN' ] . split ( '|' ) if 'CLNDN' in self . info else [ ] } )
cln_data . update ( { 'clnvi' : self . info [ 'CLNVI' ] . split ( ',' ) if 'CLNVI' in self . info else [ ] } )
try :
    sequence = self . alt_alleles [ 0 ]
except IndexError :
    sequence = self . ref_allele
allele = ClinVarAllele ( frequency = pref_freq , sequence = sequence , ** cln_data )
# A few ClinVar variants are only reported as a combination with
# other variants , and no single - variant effect is proposed . Skip these .
if not cln_data [ 'clnsig' ] :
    return [ ]
return [ allele ]


signature:
def add ( self , * names ) :
    '''Returns back a class decorator that enables registering Blox to this factory'''

implementation:
def decorator ( blok ) :
    for name in names or ( blok . __name__ , ) :
        self [ name ] = blok
    return blok
return decorator


signature:
def depricated_name ( newmethod ) :
    """Decorator for warning user of depricated functions before use .
 Args :
 newmethod ( str ) : Name of method to use instead ."""

implementation:
def decorator ( func ) :
    @ wraps ( func )
    def wrapper ( * args , ** kwargs ) :
        warnings . simplefilter ( 'always' , DeprecationWarning )
        warnings . warn ( "Function {} is depricated, please use {} instead." . format ( func . __name__ , newmethod ) , category = DeprecationWarning , stacklevel = 2 )
        warnings . simplefilter ( 'default' , DeprecationWarning )
        return func ( * args , ** kwargs )
    return wrapper
return decorator


signature:
def setDefaultRedisConnectionParams ( connectionParams ) :
    '''setDefaultRedisConnectionParams - Sets the default parameters used when connecting to Redis .
 This should be the args to redis . Redis in dict ( kwargs ) form .
 @ param connectionParams < dict > - A dict of connection parameters .
 Common keys are :
 host < str > - hostname / ip of Redis server ( default ' 127.0.0.1 ' )
 port < int > - Port number ( default 6379)
 db < int > - Redis DB number ( default 0)
 Omitting any of those keys will ensure the default value listed is used .
 This connection info will be used by default for all connections to Redis , unless explicitly set otherwise .
 The common way to override is to define REDIS _ CONNECTION _ PARAMS on a model , or use AltConnectedModel = MyModel . connectAlt ( PARAMS )
 Any omitted fields in these connection overrides will inherit the value from the global default .
 For example , if your global default connection params define host = ' example . com ' , port = 15000 , and db = 0,
 and then one of your models has
 REDIS _ CONNECTION _ PARAMS = { ' db ' : 1 }
 as an attribute , then that model ' s connection will inherit host = ' example . com " and port = 15000 but override db and use db = 1
 NOTE : Calling this function will clear the connection _ pool attribute of all stored managed connections , disconnect all managed connections ,
 and close - out the connection pool .
 It may not be safe to call this function while other threads are potentially hitting Redis ( not that it would make sense anyway . . . )
 @ see clearRedisPools for more info'''

implementation:
global _defaultRedisConnectionParams
_defaultRedisConnectionParams . clear ( )
for key , value in connectionParams . items ( ) :
    _defaultRedisConnectionParams [ key ] = value
clearRedisPools ( )


signature:
def clearRedisPools ( ) :
    '''clearRedisPools - Disconnect all managed connection pools ,
 and clear the connectiobn _ pool attribute on all stored managed connection pools .
 A " managed " connection pool is one where REDIS _ CONNECTION _ PARAMS does not define the " connection _ pool " attribute .
 If you define your own pools , IndexedRedis will use them and leave them alone .
 This method will be called automatically after calling setDefaultRedisConnectionParams .
 Otherwise , you shouldn ' t have to call it . . Maybe as some sort of disaster - recovery call . .'''

implementation:
global RedisPools
global _redisManagedConnectionParams
for pool in RedisPools . values ( ) :
    try :
        pool . disconnect ( )
    except :
        pass
for paramsList in _redisManagedConnectionParams . values ( ) :
    for params in paramsList :
        if 'connection_pool' in params :
            del params [ 'connection_pool' ]
RedisPools . clear ( )
_redisManagedConnectionParams . clear ( )


signature:
def getRedisPool ( params ) :
    '''getRedisPool - Returns and possibly also creates a Redis connection pool
 based on the REDIS _ CONNECTION _ PARAMS passed in .
 The goal of this method is to keep a small connection pool rolling
 to each unique Redis instance , otherwise during network issues etc
 python - redis will leak connections and in short - order can exhaust
 all the ports on a system . There ' s probably also some minor
 performance gain in sharing Pools .
 Will modify " params " , if " host " and / or " port " are missing , will fill
 them in with defaults , and prior to return will set " connection _ pool "
 on params , which will allow immediate return on the next call ,
 and allow access to the pool directly from the model object .
 @ param params < dict > - REDIS _ CONNECTION _ PARAMS - kwargs to redis . Redis
 @ return redis . ConnectionPool corrosponding to this unique server .'''

implementation:
global RedisPools
global _defaultRedisConnectionParams
global _redisManagedConnectionParams
if not params :
    params = _defaultRedisConnectionParams
    isDefaultParams = True
else :
    isDefaultParams = bool ( params is _defaultRedisConnectionParams )
if 'connection_pool' in params :
    return params [ 'connection_pool' ]
hashValue = hashDictOneLevel ( params )
if hashValue in RedisPools :
    params [ 'connection_pool' ] = RedisPools [ hashValue ]
    return RedisPools [ hashValue ]
# Copy the params , so that we don ' t modify the original dict
if not isDefaultParams :
    origParams = params
    params = copy . copy ( params )
else :
    origParams = params
checkAgain = False
if 'host' not in params :
    if not isDefaultParams and 'host' in _defaultRedisConnectionParams :
        params [ 'host' ] = _defaultRedisConnectionParams [ 'host' ]
    else :
        params [ 'host' ] = '127.0.0.1'
    checkAgain = True
if 'port' not in params :
    if not isDefaultParams and 'port' in _defaultRedisConnectionParams :
        params [ 'port' ] = _defaultRedisConnectionParams [ 'port' ]
    else :
        params [ 'port' ] = 6379
    checkAgain = True
if 'db' not in params :
    if not isDefaultParams and 'db' in _defaultRedisConnectionParams :
        params [ 'db' ] = _defaultRedisConnectionParams [ 'db' ]
    else :
        params [ 'db' ] = 0
    checkAgain = True
if not isDefaultParams :
    otherGlobalKeys = set ( _defaultRedisConnectionParams . keys ( ) ) - set ( params . keys ( ) )
    for otherKey in otherGlobalKeys :
        if otherKey == 'connection_pool' :
            continue
        params [ otherKey ] = _defaultRedisConnectionParams [ otherKey ]
        checkAgain = True
if checkAgain :
    hashValue = hashDictOneLevel ( params )
    if hashValue in RedisPools :
        params [ 'connection_pool' ] = RedisPools [ hashValue ]
        return RedisPools [ hashValue ]
connectionPool = redis . ConnectionPool ( ** params )
origParams [ 'connection_pool' ] = params [ 'connection_pool' ] = connectionPool
RedisPools [ hashValue ] = connectionPool
# Add the original as a " managed " redis connection ( they did not provide their own pool )
# such that if the defaults change , we make sure to re - inherit any keys , and can disconnect
# from clearRedisPools
origParamsHash = hashDictOneLevel ( origParams )
if origParamsHash not in _redisManagedConnectionParams :
    _redisManagedConnectionParams [ origParamsHash ] = [ origParams ]
elif origParams not in _redisManagedConnectionParams [ origParamsHash ] :
    _redisManagedConnectionParams [ origParamsHash ] . append ( origParams )
return connectionPool


signature:
def asDict ( self , includeMeta = False , forStorage = False , strKeys = False ) :
    '''toDict / asDict - Get a dictionary representation of this model .
 @ param includeMeta - Include metadata in return . For now , this is only pk stored as " _ id "
 @ param convertValueTypes < bool > - default True . If False , fields with fieldValue defined will be converted to that type .
 Use True when saving , etc , as native type is always either str or bytes .
 @ param strKeys < bool > Default False - If True , just the string value of the field name will be used as the key .
 Otherwise , the IRField itself will be ( although represented and indexed by string )
 @ return - Dictionary reprensetation of this object and all fields'''

implementation:
ret = { }
for thisField in self . FIELDS :
    val = object . __getattribute__ ( self , thisField )
    if forStorage is True :
        val = thisField . toStorage ( val )
    if strKeys :
        ret [ str ( thisField ) ] = val
    else :
        ret [ thisField ] = val
if includeMeta is True :
    ret [ '_id' ] = getattr ( self , '_id' , None )
return ret


signature:
def pprint ( self , stream = None ) :
    '''pprint - Pretty - print a dict representation of this object .
 @ param stream < file / None > - Either a stream to output , or None to default to sys . stdout'''

implementation: pprint . pprint ( self . asDict ( includeMeta = True , forStorage = False , strKeys = True ) , stream = stream )


signature:
def hasUnsavedChanges ( self , cascadeObjects = False ) :
    '''hasUnsavedChanges - Check if any unsaved changes are present in this model , or if it has never been saved .
 @ param cascadeObjects < bool > default False , if True will check if any foreign linked objects themselves have unsaved changes ( recursively ) .
 Otherwise , will just check if the pk has changed .
 @ return < bool > - True if any fields have changed since last fetch , or if never saved . Otherwise , False'''

implementation:
if not self . _id or not self . _origData :
    return True
for thisField in self . FIELDS :
    thisVal = object . __getattribute__ ( self , thisField )
    if self . _origData . get ( thisField , '' ) != thisVal :
        return True
    if cascadeObjects is True and issubclass ( thisField . __class__ , IRForeignLinkFieldBase ) :
        if thisVal . objHasUnsavedChanges ( ) :
            return True
return False


signature:
def getUpdatedFields ( self , cascadeObjects = False ) :
    '''getUpdatedFields - See changed fields .
 @ param cascadeObjects < bool > default False , if True will check if any foreign linked objects themselves have unsaved changes ( recursively ) .
 Otherwise , will just check if the pk has changed .
 @ return - a dictionary of fieldName : tuple ( old , new ) .
 fieldName may be a string or may implement IRField ( which implements string , and can be used just like a string )'''

implementation:
updatedFields = { }
for thisField in self . FIELDS :
    thisVal = object . __getattribute__ ( self , thisField )
    if self . _origData . get ( thisField , '' ) != thisVal :
        updatedFields [ thisField ] = ( self . _origData [ thisField ] , thisVal )
    if cascadeObjects is True and issubclass ( thisField . __class__ , IRForeignLinkFieldBase ) and thisVal . objHasUnsavedChanges ( ) :
        updatedFields [ thisField ] = ( self . _origData [ thisField ] , thisVal )
return updatedFields


signature:
def diff ( firstObj , otherObj , includeMeta = False ) :
    '''diff - Compare the field values on two IndexedRedisModels .
 @ param firstObj < IndexedRedisModel instance > - First object ( or self )
 @ param otherObj < IndexedRedisModel instance > - Second object
 @ param includeMeta < bool > - If meta information ( like pk ) should be in the diff results .
 @ return < dict > - Dict of ' field ' : ( value _ firstObjForField , value _ otherObjForField ) .
 Keys are names of fields with different values .
 Value is a tuple of ( value _ firstObjForField , value _ otherObjForField )
 Can be called statically , like : IndexedRedisModel . diff ( obj1 , obj2 )
 or in reference to an obj : obj1 . diff ( obj2)'''

implementation:
if not isIndexedRedisModel ( firstObj ) :
    raise ValueError ( 'Type < %s > does not extend IndexedRedisModel.' % ( type ( firstObj ) . __name__ , ) )
if not isIndexedRedisModel ( otherObj ) :
    raise ValueError ( 'Type < %s > does not extend IndexedRedisModel.' % ( type ( otherObj ) . __name__ , ) )
firstObj . validateModel ( )
otherObj . validateModel ( )
# Types may not match , but could be subclass , special copy class ( like connectAlt ) , etc .
# So check if FIELDS matches , and if so , we can continue .
if getattr ( firstObj , 'FIELDS' ) != getattr ( otherObj , 'FIELDS' ) : # NOTE : Maybe we should iterate here and compare just that field types and names match ?
# In case a copy changes a default or something , we would still be able to diff . .
    raise ValueError ( 'Cannot compare  < %s > and < %s > . Must be same model OR have equal FIELDS.' % ( firstObj . __class__ , otherObj . __class__ ) )
diffFields = { }
for thisField in firstObj . FIELDS :
    thisFieldStr = str ( thisField )
    firstVal = object . __getattribute__ ( firstObj , thisFieldStr )
    otherVal = object . __getattribute__ ( otherObj , thisFieldStr )
    if firstVal != otherVal :
        diffFields [ thisFieldStr ] = ( ( firstVal , otherVal ) )
if includeMeta :
    firstPk = firstObj . getPk ( )
    otherPk = otherObj . getPk ( )
    if firstPk != otherPk :
        diffFields [ '_id' ] = ( firstPk , otherPk )
return diffFields


signature:
def save ( self , cascadeSave = True ) :
    '''save - Save this object .
 Will perform an " insert " if this object had not been saved before ,
 otherwise will update JUST the fields changed on THIS INSTANCE of the model .
 i . e . If you have two processes fetch the same object and change different fields , they will not overwrite
 eachother , but only save the ones each process changed .
 If you want to save multiple objects of type MyModel in a single transaction ,
 and you have those objects in a list , myObjs , you can do the following :
 MyModel . saver . save ( myObjs )
 @ param cascadeSave < bool > Default True - If True , any Foreign models linked as attributes that have been altered
 or created will be saved with this object . If False , only this object ( and the reference to an already - saved foreign model ) will be saved .
 @ see # IndexedRedisSave . save
 @ return < list > - Single element list , id of saved object ( if successful )'''

implementation:
saver = IndexedRedisSave ( self . __class__ )
return saver . save ( self , cascadeSave = cascadeSave )


signature:
def reset ( cls , newObjs ) :
    '''reset - Remove all stored data associated with this model ( i . e . all objects of this type ) ,
 and then save all the provided objects in # newObjs , all in one atomic transaction .
 Use this method to move from one complete set of objects to another , where any querying applications
 will only see the complete before or complete after .
 @ param newObjs list < IndexedRedisModel objs > - A list of objects that will replace the current dataset
 To just replace a specific subset of objects in a single transaction , you can do MyModel . saver . save ( objs )
 and just the objs in " objs " will be inserted / updated in one atomic step .
 This method , on the other hand , will delete all previous objects and add the newly provided objects in a single atomic step ,
 and also reset the primary key ID generator
 @ return list < int > - The new primary keys associated with each object ( same order as provided # newObjs list )'''

implementation:
conn = cls . objects . _get_new_connection ( )
transaction = conn . pipeline ( )
transaction . eval ( """
		local matchingKeys = redis.call('KEYS', '%s*')

		for _,key in ipairs(matchingKeys) do
			redis.call('DEL', key)
		end
		""" % ( '' . join ( [ INDEXED_REDIS_PREFIX , cls . KEY_NAME , ':' ] ) , ) , 0 )
saver = IndexedRedisSave ( cls )
nextID = 1
for newObj in newObjs :
    saver . save ( newObj , False , forceID = nextID , conn = transaction )
    nextID += 1
transaction . set ( saver . _get_next_id_key ( ) , nextID )
transaction . execute ( )
return list ( range ( 1 , nextID , 1 ) )


signature:
def hasSameValues ( self , other , cascadeObject = True ) :
    '''hasSameValues - Check if this and another model have the same fields and values .
 This does NOT include id , so the models can have the same values but be different objects in the database .
 @ param other < IndexedRedisModel > - Another model
 @ param cascadeObject < bool > default True - If True , foreign link values with changes will be considered a difference .
 Otherwise , only the immediate values are checked .
 @ return < bool > - True if all fields have the same value , otherwise False'''

implementation:
if self . FIELDS != other . FIELDS :
    return False
oga = object . __getattribute__
for field in self . FIELDS :
    thisVal = oga ( self , field )
    otherVal = oga ( other , field )
    if thisVal != otherVal :
        return False
    if cascadeObject is True and issubclass ( field . __class__ , IRForeignLinkFieldBase ) :
        if thisVal and thisVal . isFetched ( ) :
            if otherVal and otherVal . isFetched ( ) :
                theseForeign = thisVal . getObjs ( )
                othersForeign = otherVal . getObjs ( )
                for i in range ( len ( theseForeign ) ) :
                    if not theseForeign [ i ] . hasSameValues ( othersForeign [ i ] ) :
                        return False
            else :
                theseForeign = thisVal . getObjs ( )
                for i in range ( len ( theseForeign ) ) :
                    if theseForeign [ i ] . hasUnsavedChanges ( cascadeObjects = True ) :
                        return False
        else :
            if otherVal and otherVal . isFetched ( ) :
                othersForeign = otherVal . getObjs ( )
                for i in range ( len ( othersForeign ) ) :
                    if othersForeign [ i ] . hasUnsavedChanges ( cascadeObjects = True ) :
                        return False
return True


signature:
def copy ( self , copyPrimaryKey = False , copyValues = False ) :
    '''copy - Copies this object .
 @ param copyPrimaryKey < bool > default False - If True , any changes to the copy will save over - top the existing entry in Redis .
 If False , only the data is copied , and nothing is saved .
 @ param copyValues < bool > default False - If True , every field value on this object will be explicitly copied . If False ,
 an object will be created with the same values , and depending on the type may share the same reference .
 This is the difference between a copy and a deepcopy .
 @ return < IndexedRedisModel > - Copy of this object , per above
 If you need a copy that IS linked , @ see IndexedRedisModel . copy'''

implementation:
cpy = self . __class__ ( ** self . asDict ( copyPrimaryKey , forStorage = False ) )
if copyValues is True :
    for fieldName in cpy . FIELDS :
        setattr ( cpy , fieldName , copy . deepcopy ( getattr ( cpy , fieldName ) ) )
return cpy


signature:
def saveToExternal ( self , redisCon ) :
    '''saveToExternal - Saves this object to a different Redis than that specified by REDIS _ CONNECTION _ PARAMS on this model .
 @ param redisCon < dict / redis . Redis > - Either a dict of connection params , a la REDIS _ CONNECTION _ PARAMS , or an existing Redis connection .
 If you are doing a lot of bulk copies , it is recommended that you create a Redis connection and pass it in rather than establish a new
 connection with each call .
 @ note - You will generate a new primary key relative to the external Redis environment . If you need to reference a " shared " primary key , it is better
 to use an indexed field than the internal pk .'''

implementation:
if type ( redisCon ) == dict :
    conn = redis . Redis ( ** redisCon )
elif hasattr ( conn , '__class__' ) and issubclass ( conn . __class__ , redis . Redis ) :
    conn = redisCon
else :
    raise ValueError ( 'saveToExternal "redisCon" param must either be a dictionary of connection parameters, or redis.Redis, or extension thereof' )
saver = self . saver
# Fetch next PK from external
forceID = saver . _getNextID ( conn )
# Redundant because of changes in save method
myCopy = self . copy ( False )
return saver . save ( myCopy , usePipeline = True , forceID = forceID , conn = conn )


signature:
def reload ( self , cascadeObjects = True ) :
    '''reload - Reload this object from the database , overriding any local changes and merging in any updates .
 @ param cascadeObjects < bool > Default True . If True , foreign - linked objects will be reloaded if their values have changed
 since last save / fetch . If False , only if the pk changed will the foreign linked objects be reloaded .
 @ raises KeyError - if this object has not been saved ( no primary key )
 @ return - Dict with the keys that were updated . Key is field name that was updated ,
 and value is tuple of ( old value , new value ) .
 NOTE : Currently , this will cause a fetch of all Foreign Link objects , one level'''

implementation:
_id = self . _id
if not _id :
    raise KeyError ( 'Object has never been saved! Cannot reload.' )
currentData = self . asDict ( False , forStorage = False )
# Get the object , and compare the unconverted " asDict " repr .
# If any changes , we will apply the already - convered value from
# the object , but we compare the unconverted values ( what ' s in the DB ) .
newDataObj = self . objects . get ( _id )
if not newDataObj :
    raise KeyError ( 'Object with id=%d is not in database. Cannot reload.' % ( _id , ) )
newData = newDataObj . asDict ( False , forStorage = False )
if currentData == newData and not self . foreignFields :
    return [ ]
updatedFields = { }
for thisField , newValue in newData . items ( ) :
    defaultValue = thisField . getDefaultValue ( )
    currentValue = currentData . get ( thisField , defaultValue )
    fieldIsUpdated = False
    if currentValue != newValue :
        fieldIsUpdated = True
    elif cascadeObjects is True and issubclass ( thisField . __class__ , IRForeignLinkFieldBase ) : # If we are cascading objects , and at this point the pk is the same
        if currentValue . isFetched ( ) : # If we have fetched the current set , we might need to update ( pks already match )
            oldObjs = currentValue . getObjs ( )
            newObjs = newValue . getObjs ( )
            if oldObjs != newObjs : # This will check using _ _ eq _ _ , so one - level including pk
                fieldIsUpdated = True
            else : # Use hasSameValues with cascadeObjects = True to scan past one level
                for i in range ( len ( oldObjs ) ) :
                    if not oldObjs [ i ] . hasSameValues ( newObjs [ i ] , cascadeObjects = True ) :
                        fieldIsUpdated = True
                        break
    if fieldIsUpdated is True : # Use " converted " values in the updatedFields dict , and apply on the object .
        updatedFields [ thisField ] = ( currentValue , newValue )
        setattr ( self , thisField , newValue )
        self . _origData [ thisField ] = newDataObj . _origData [ thisField ]
return updatedFields


signature:
def copyModel ( mdl ) :
    '''copyModel - Copy this model , and return that copy .
 The copied model will have all the same data , but will have a fresh instance of the FIELDS array and all members ,
 and the INDEXED _ FIELDS array .
 This is useful for converting , like changing field types or whatever , where you can load from one model and save into the other .
 @ return < IndexedRedisModel > - A copy class of this model class with a unique name .'''

implementation:
copyNum = _modelCopyMap [ mdl ]
_modelCopyMap [ mdl ] += 1
mdlCopy = type ( mdl . __name__ + '_Copy' + str ( copyNum ) , mdl . __bases__ , copy . deepcopy ( dict ( mdl . __dict__ ) ) )
mdlCopy . FIELDS = [ field . copy ( ) for field in mdl . FIELDS ]
mdlCopy . INDEXED_FIELDS = [ str ( idxField ) for idxField in mdl . INDEXED_FIELDS ]
# Make sure they didn ' t do INDEXED _ FIELDS = FIELDS or something wacky ,
# so do a comprehension of str on these to make sure we only get names
mdlCopy . validateModel ( )
return mdlCopy


signature:
def validateModel ( model ) :
    '''validateModel - Class method that validates a given model is implemented correctly . Will only be validated once , on first model instantiation .
 @ param model - Implicit of own class
 @ return - True
 @ raises - InvalidModelException if there is a problem with the model , and the message contains relevant information .'''

implementation:
if model == IndexedRedisModel :
    import re
    if re . match ( '.*pydoc(|[\d]|[\d][\.][\d])([\.]py(|[co])){0,1}$' , sys . argv [ 0 ] ) :
        return
    raise ValueError ( 'Cannot use IndexedRedisModel directly. You must implement this class for your model.' )
global validatedModels
keyName = model . KEY_NAME
if not keyName :
    raise InvalidModelException ( '"%s" has no KEY_NAME defined.' % ( str ( model . __name__ ) , ) )
if model in validatedModels :
    return True
failedValidationStr = '"%s" Failed Model Validation:' % ( str ( model . __name__ ) , )
# Convert items in model to set
# model . FIELDS = set ( model . FIELDS )
fieldSet = set ( model . FIELDS )
indexedFieldSet = set ( model . INDEXED_FIELDS )
if not fieldSet :
    raise InvalidModelException ( '%s No fields defined. Please populate the FIELDS array with a list of field names' % ( failedValidationStr , ) )
if hasattr ( model , 'BASE64_FIELDS' ) :
    raise InvalidModelException ( 'BASE64_FIELDS is no longer supported since IndexedRedis 5.0.0 . Use IndexedRedis.fields.IRBase64Field in the FIELDS array for the same functionality.' )
if hasattr ( model , 'BINARY_FIELDS' ) :
    raise InvalidModelException ( 'BINARY_FIELDS is no longer supported since IndexedRedis 5.0.0 . Use IndexedRedis.fields.IRBytesField in the FIELDS array for the same functionality, use IRBytesField for same functionality. Use IRField(valueType=bytes) for python-3 only support. Use IRRawField to perform no conversion at all.' )
newFields = [ ]
updatedFields = [ ]
mustUpdateFields = False
foreignFields = [ ]
for thisField in fieldSet :
    if thisField == '_id' :
        raise InvalidModelException ( '%s You cannot have a field named _id, it is reserved for the primary key.' % ( failedValidationStr , ) )
    # XXX : Is this ascii requirement still needed since all is unicode now ?
    try :
        codecs . ascii_encode ( thisField )
    except UnicodeDecodeError as e :
        raise InvalidModelException ( '%s All field names must be ascii-encodable. "%s" was not. Error was: %s' % ( failedValidationStr , to_unicode ( thisField ) , str ( e ) ) )
    if issubclass ( thisField . __class__ , IRForeignLinkFieldBase ) :
        foreignFields . append ( thisField )
    # If a classic string field , convert to IRClassicField
    if issubclass ( thisField . __class__ , IRField ) :
        newFields . append ( thisField )
    else :
        mustUpdateFields = True
        newField = IRClassicField ( thisField )
        newFields . append ( newField )
        updatedFields . append ( thisField )
        thisField = newField
    if str ( thisField ) == '' :
        raise InvalidModelException ( '%s Field defined without a name, or name was an empty string. Type=%s  Field is:  %s' % ( failedValidationStr , str ( type ( thisField ) ) , repr ( thisField ) ) )
    if thisField in indexedFieldSet and thisField . CAN_INDEX is False :
        raise InvalidModelException ( '%s Field Type %s - (%s) cannot be indexed.' % ( failedValidationStr , str ( thisField . __class__ . __name__ ) , repr ( thisField ) ) )
    if hasattr ( IndexedRedisModel , thisField ) is True :
        raise InvalidModelException ( '%s Field name %s is a reserved attribute on IndexedRedisModel.' % ( failedValidationStr , str ( thisField ) ) )
if mustUpdateFields is True :
    model . FIELDS = newFields
    deprecatedMessage ( 'Model "%s" contains plain-string fields. These have been converted to IRClassicField objects to retain the same functionality. plain-string fields will be removed in a future version. The converted fields are: %s' % ( model . __name__ , repr ( updatedFields ) ) , 'UPDATED_FIELDS_' + model . __name__ )
model . FIELDS = KeyList ( model . FIELDS )
if bool ( indexedFieldSet - fieldSet ) :
    raise InvalidModelException ( '%s All INDEXED_FIELDS must also be present in FIELDS. %s exist only in INDEXED_FIELDS' % ( failedValidationStr , str ( list ( indexedFieldSet - fieldSet ) ) , ) )
model . foreignFields = foreignFields
validatedModels . add ( model )
return True


signature:
def connectAlt ( cls , redisConnectionParams ) :
    '''connectAlt - Create a class of this model which will use an alternate connection than the one specified by REDIS _ CONNECTION _ PARAMS on this model .
 @ param redisConnectionParams < dict > - Dictionary of arguments to redis . Redis , same as REDIS _ CONNECTION _ PARAMS .
 @ return - A class that can be used in all the same ways as the existing IndexedRedisModel , but that connects to a different instance .
 The fields and key will be the same here , but the connection will be different . use # copyModel if you want an independent class for the model'''

implementation:
if not isinstance ( redisConnectionParams , dict ) :
    raise ValueError ( 'redisConnectionParams must be a dictionary!' )
hashVal = hashDictOneLevel ( redisConnectionParams )
modelDictCopy = copy . deepcopy ( dict ( cls . __dict__ ) )
modelDictCopy [ 'REDIS_CONNECTION_PARAMS' ] = redisConnectionParams
ConnectedIndexedRedisModel = type ( 'AltConnect' + cls . __name__ + str ( hashVal ) , cls . __bases__ , modelDictCopy )
return ConnectedIndexedRedisModel


signature:
def _get_new_connection ( self ) :
    '''_ get _ new _ connection - Get a new connection
 internal'''

implementation:
pool = getRedisPool ( self . mdl . REDIS_CONNECTION_PARAMS )
return redis . Redis ( connection_pool = pool )


signature:
def _get_connection ( self ) :
    '''_ get _ connection - Maybe get a new connection , or reuse if passed in .
 Will share a connection with a model
 internal'''

implementation:
if self . _connection is None :
    self . _connection = self . _get_new_connection ( )
return self . _connection


signature:
def _add_id_to_keys ( self , pk , conn = None ) :
    '''_ add _ id _ to _ keys - Adds primary key to table
 internal'''

implementation:
if conn is None :
    conn = self . _get_connection ( )
conn . sadd ( self . _get_ids_key ( ) , pk )


signature:
def _rem_id_from_keys ( self , pk , conn = None ) :
    '''_ rem _ id _ from _ keys - Remove primary key from table
 internal'''

implementation:
if conn is None :
    conn = self . _get_connection ( )
conn . srem ( self . _get_ids_key ( ) , pk )


signature:
def _add_id_to_index ( self , indexedField , pk , val , conn = None ) :
    '''_ add _ id _ to _ index - Adds an id to an index
 internal'''

implementation:
if conn is None :
    conn = self . _get_connection ( )
conn . sadd ( self . _get_key_for_index ( indexedField , val ) , pk )


signature:
def _rem_id_from_index ( self , indexedField , pk , val , conn = None ) :
    '''_ rem _ id _ from _ index - Removes an id from an index
 internal'''

implementation:
if conn is None :
    conn = self . _get_connection ( )
conn . srem ( self . _get_key_for_index ( indexedField , val ) , pk )


signature:
def _get_key_for_index ( self , indexedField , val ) :
    '''_ get _ key _ for _ index - Returns the key name that would hold the indexes on a value
 Internal - does not validate that indexedFields is actually indexed . Trusts you . Don ' t let it down .
 @ param indexedField - string of field name
 @ param val - Value of field
 @ return - Key name string , potentially hashed .'''

implementation:
# If provided an IRField , use the toIndex from that ( to support compat _ methods
if hasattr ( indexedField , 'toIndex' ) :
    val = indexedField . toIndex ( val )
else : # Otherwise , look up the indexed field from the model
    val = self . fields [ indexedField ] . toIndex ( val )
return '' . join ( [ INDEXED_REDIS_PREFIX , self . keyName , ':idx:' , indexedField , ':' , val ] )


signature:
def _compat_get_str_key_for_index ( self , indexedField , val ) :
    '''_ compat _ get _ str _ key _ for _ index - Return the key name as a string , even if it is a hashed index field .
 This is used in converting unhashed fields to a hashed index ( called by _ compat _ rem _ str _ id _ from _ index which is called by compat _ convertHashedIndexes )
 @ param inde
 @ param indexedField - string of field name
 @ param val - Value of field
 @ return - Key name string , always a string regardless of hash'''

implementation: return '' . join ( [ INDEXED_REDIS_PREFIX , self . keyName , ':idx:' , indexedField , ':' , getattr ( indexedField , 'toStorage' , to_unicode ) ( val ) ] )


signature:
def _compat_rem_str_id_from_index ( self , indexedField , pk , val , conn = None ) :
    '''_ compat _ rem _ str _ id _ from _ index - Used in compat _ convertHashedIndexes to remove the old string repr of a field ,
 in order to later add the hashed value ,'''

implementation:
if conn is None :
    conn = self . _get_connection ( )
conn . srem ( self . _compat_get_str_key_for_index ( indexedField , val ) , pk )


signature:
def _peekNextID ( self , conn = None ) :
    '''_ peekNextID - Look at , but don ' t increment the primary key for this model .
 Internal .
 @ return int - next pk'''

implementation:
if conn is None :
    conn = self . _get_connection ( )
return to_unicode ( conn . get ( self . _get_next_id_key ( ) ) or 0 )


signature:
def _getNextID ( self , conn = None ) :
    '''_ getNextID - Get ( and increment ) the next primary key for this model .
 If you don ' t want to increment , @ see _ peekNextID .
 Internal .
 This is done automatically on save . No need to call it .
 @ return int - next pk'''

implementation:
if conn is None :
    conn = self . _get_connection ( )
return int ( conn . incr ( self . _get_next_id_key ( ) ) )


signature:
def filter ( self , ** kwargs ) :
    '''filter - Add filters based on INDEXED _ FIELDS having or not having a value .
 Note , no objects are actually fetched until . all ( ) is called
 Use the field name [ model . objects . filter ( some _ field = ' value ' ) ] to filter on items containing that value .
 Use the field name suffxed with ' _ _ ne ' for a negation filter [ model . objects . filter ( some _ field _ _ ne = ' value ' ) ]
 Example :
 query = Model . objects . filter ( field1 = ' value ' , field2 = ' othervalue ' )
 objs1 = query . filter ( something _ _ ne = ' value ' ) . all ( )
 objs2 = query . filter ( something _ _ ne = 7 ) . all ( )
 @ returns - A copy of this object , with the additional filters . If you want to work inline on this object instead , use the filterInline method .'''

implementation:
selfCopy = self . __copy__ ( )
return IndexedRedisQuery . _filter ( selfCopy , ** kwargs )


signature:
def _filter ( filterObj , ** kwargs ) :
    '''Internal for handling filters ; the guts of . filter and . filterInline'''

implementation:
for key , value in kwargs . items ( ) :
    if key . endswith ( '__ne' ) :
        notFilter = True
        key = key [ : - 4 ]
    else :
        notFilter = False
    if key not in filterObj . indexedFields :
        raise ValueError ( 'Field "' + key + '" is not in INDEXED_FIELDS array. Filtering is only supported on indexed fields.' )
    if notFilter is False :
        filterObj . filters . append ( ( key , value ) )
    else :
        filterObj . notFilters . append ( ( key , value ) )
return filterObj


signature:
def count ( self ) :
    '''count - gets the number of records matching the filter criteria
 Example :
 theCount = Model . objects . filter ( field1 = ' value ' ) . count ( )'''

implementation:
conn = self . _get_connection ( )
numFilters = len ( self . filters )
numNotFilters = len ( self . notFilters )
if numFilters + numNotFilters == 0 :
    return conn . scard ( self . _get_ids_key ( ) )
if numNotFilters == 0 :
    if numFilters == 1 :
        ( filterFieldName , filterValue ) = self . filters [ 0 ]
        return conn . scard ( self . _get_key_for_index ( filterFieldName , filterValue ) )
    indexKeys = [ self . _get_key_for_index ( filterFieldName , filterValue ) for filterFieldName , filterValue in self . filters ]
    return len ( conn . sinter ( indexKeys ) )
notIndexKeys = [ self . _get_key_for_index ( filterFieldName , filterValue ) for filterFieldName , filterValue in self . notFilters ]
if numFilters == 0 :
    return len ( conn . sdiff ( self . _get_ids_key ( ) , * notIndexKeys ) )
indexKeys = [ self . _get_key_for_index ( filterFieldName , filterValue ) for filterFieldName , filterValue in self . filters ]
tempKey = self . _getTempKey ( )
pipeline = conn . pipeline ( )
pipeline . sinterstore ( tempKey , * indexKeys )
pipeline . sdiff ( tempKey , * notIndexKeys )
pipeline . delete ( tempKey )
pks = pipeline . execute ( ) [ 1 ]
# sdiff
return len ( pks )


signature:
def exists ( self , pk ) :
    '''exists - Tests whether a record holding the given primary key exists .
 @ param pk - Primary key ( see getPk method )
 Example usage : Waiting for an object to be deleted without fetching the object or running a filter .
 This is a very cheap operation .
 @ return < bool > - True if object with given pk exists , otherwise False'''

implementation:
conn = self . _get_connection ( )
key = self . _get_key_for_id ( pk )
return conn . exists ( key )


signature:
def getPrimaryKeys ( self , sortByAge = False ) :
    '''getPrimaryKeys - Returns all primary keys matching current filterset .
 @ param sortByAge < bool > - If False , return will be a set and may not be ordered .
 If True , return will be a list and is guarenteed to represent objects oldest - > newest
 @ return < set > - A set of all primary keys associated with current filters .'''

implementation:
conn = self . _get_connection ( )
# Apply filters , and return object
numFilters = len ( self . filters )
numNotFilters = len ( self . notFilters )
if numFilters + numNotFilters == 0 : # No filters , get all .
    conn = self . _get_connection ( )
    matchedKeys = conn . smembers ( self . _get_ids_key ( ) )
elif numNotFilters == 0 : # Only Inclusive
    if numFilters == 1 : # Only one filter , get members of that index key
        ( filterFieldName , filterValue ) = self . filters [ 0 ]
        matchedKeys = conn . smembers ( self . _get_key_for_index ( filterFieldName , filterValue ) )
    else : # Several filters , intersect the index keys
        indexKeys = [ self . _get_key_for_index ( filterFieldName , filterValue ) for filterFieldName , filterValue in self . filters ]
        matchedKeys = conn . sinter ( indexKeys )
else : # Some negative filters present
    notIndexKeys = [ self . _get_key_for_index ( filterFieldName , filterValue ) for filterFieldName , filterValue in self . notFilters ]
    if numFilters == 0 : # Only negative , diff against all keys
        matchedKeys = conn . sdiff ( self . _get_ids_key ( ) , * notIndexKeys )
    else : # Negative and positive . Use pipeline , find all positive intersections , and remove negative matches
        indexKeys = [ self . _get_key_for_index ( filterFieldName , filterValue ) for filterFieldName , filterValue in self . filters ]
        tempKey = self . _getTempKey ( )
        pipeline = conn . pipeline ( )
        pipeline . sinterstore ( tempKey , * indexKeys )
        pipeline . sdiff ( tempKey , * notIndexKeys )
        pipeline . delete ( tempKey )
        matchedKeys = pipeline . execute ( ) [ 1 ]
# sdiff
matchedKeys = [ int ( _key ) for _key in matchedKeys ]
if sortByAge is False :
    return list ( matchedKeys )
else :
    matchedKeys = list ( matchedKeys )
    matchedKeys . sort ( )
    return matchedKeys


signature:
def all ( self , cascadeFetch = False ) :
    '''all - Get the underlying objects which match the filter criteria .
 Example : objs = Model . objects . filter ( field1 = ' value ' , field2 = ' value2 ' ) . all ( )
 @ param cascadeFetch < bool > Default False , If True , all Foreign objects associated with this model
 will be fetched immediately . If False , foreign objects will be fetched on - access .
 @ return - Objects of the Model instance associated with this query .'''

implementation:
matchedKeys = self . getPrimaryKeys ( )
if matchedKeys :
    return self . getMultiple ( matchedKeys , cascadeFetch = cascadeFetch )
return IRQueryableList ( [ ] , mdl = self . mdl )


signature:
def allByAge ( self , cascadeFetch = False ) :
    '''allByAge - Get the underlying objects which match the filter criteria , ordered oldest - > newest
 If you are doing a queue or just need the head / tail , consider . first ( ) and . last ( ) instead .
 @ param cascadeFetch < bool > Default False , If True , all Foreign objects associated with this model
 will be fetched immediately . If False , foreign objects will be fetched on - access .
 @ return - Objects of the Model instance associated with this query , sorted oldest - > newest'''

implementation:
matchedKeys = self . getPrimaryKeys ( sortByAge = True )
if matchedKeys :
    return self . getMultiple ( matchedKeys , cascadeFetch = cascadeFetch )
return IRQueryableList ( [ ] , mdl = self . mdl )


signature:
def allOnlyFields ( self , fields , cascadeFetch = False ) :
    '''allOnlyFields - Get the objects which match the filter criteria , only fetching given fields .
 @ param fields - List of fields to fetch
 @ param cascadeFetch < bool > Default False , If True , all Foreign objects associated with this model
 will be fetched immediately . If False , foreign objects will be fetched on - access .
 @ return - Partial objects with only the given fields fetched'''

implementation:
matchedKeys = self . getPrimaryKeys ( )
if matchedKeys :
    return self . getMultipleOnlyFields ( matchedKeys , fields , cascadeFetch = cascadeFetch )
return IRQueryableList ( [ ] , mdl = self . mdl )


signature:
def allOnlyIndexedFields ( self ) :
    '''allOnlyIndexedFields - Get the objects which match the filter criteria , only fetching indexed fields .
 @ return - Partial objects with only the indexed fields fetched'''

implementation:
matchedKeys = self . getPrimaryKeys ( )
if matchedKeys :
    return self . getMultipleOnlyIndexedFields ( matchedKeys )
return IRQueryableList ( [ ] , mdl = self . mdl )


signature:
def first ( self , cascadeFetch = False ) :
    '''First - Returns the oldest record ( lowerst primary key ) with current filters .
 This makes an efficient queue , as it only fetches a single object .
 @ param cascadeFetch < bool > Default False , If True , all Foreign objects associated with this model
 will be fetched immediately . If False , foreign objects will be fetched on - access .
 @ return - Instance of Model object , or None if no items match current filters'''

implementation:
obj = None
matchedKeys = self . getPrimaryKeys ( sortByAge = True )
if matchedKeys : # Loop so we don ' t return None when there are items , if item is deleted between getting key and getting obj
    while matchedKeys and obj is None :
        obj = self . get ( matchedKeys . pop ( 0 ) , cascadeFetch = cascadeFetch )
return obj


signature:
def random ( self , cascadeFetch = False ) :
    '''Random - Returns a random record in current filterset .
 @ param cascadeFetch < bool > Default False , If True , all Foreign objects associated with this model
 will be fetched immediately . If False , foreign objects will be fetched on - access .
 @ return - Instance of Model object , or None if no items math current filters'''

implementation:
matchedKeys = list ( self . getPrimaryKeys ( ) )
obj = None
# Loop so we don ' t return None when there are items , if item is deleted between getting key and getting obj
while matchedKeys and not obj :
    key = matchedKeys . pop ( random . randint ( 0 , len ( matchedKeys ) - 1 ) )
    obj = self . get ( key , cascadeFetch = cascadeFetch )
return obj


signature:
def delete ( self ) :
    '''delete - Deletes all entries matching the filter criteria'''

implementation:
if self . filters or self . notFilters :
    return self . mdl . deleter . deleteMultiple ( self . allOnlyIndexedFields ( ) )
return self . mdl . deleter . destroyModel ( )


signature:
def get ( self , pk , cascadeFetch = False ) :
    '''get - Get a single value with the internal primary key .
 @ param cascadeFetch < bool > Default False , If True , all Foreign objects associated with this model
 will be fetched immediately . If False , foreign objects will be fetched on - access .
 @ param pk - internal primary key ( can be found via . getPk ( ) on an item )'''

implementation:
conn = self . _get_connection ( )
key = self . _get_key_for_id ( pk )
res = conn . hgetall ( key )
if type ( res ) != dict or not len ( res . keys ( ) ) :
    return None
res [ '_id' ] = pk
ret = self . _redisResultToObj ( res )
if cascadeFetch is True :
    self . _doCascadeFetch ( ret )
return ret


signature:
def _doCascadeFetch ( obj ) :
    '''_ doCascadeFetch - Takes an object and performs a cascading fetch on all foreign links , and all theirs , and so on .
 @ param obj < IndexedRedisModel > - A fetched model'''

implementation:
obj . validateModel ( )
if not obj . foreignFields :
    return
# NOTE : Currently this fetches using one transaction per object . Implementation for actual resolution is in
# IndexedRedisModel . _ _ getattribute _ _
for foreignField in obj . foreignFields :
    subObjsData = object . __getattribute__ ( obj , foreignField )
    if not subObjsData :
        setattr ( obj , str ( foreignField ) , irNull )
        continue
    subObjs = subObjsData . getObjs ( )
    for subObj in subObjs :
        if isIndexedRedisModel ( subObj ) :
            IndexedRedisQuery . _doCascadeFetch ( subObj )


signature:
def getMultiple ( self , pks , cascadeFetch = False ) :
    '''getMultiple - Gets multiple objects with a single atomic operation
 @ param cascadeFetch < bool > Default False , If True , all Foreign objects associated with this model
 will be fetched immediately . If False , foreign objects will be fetched on - access .
 @ param pks - list of internal keys'''

implementation:
if type ( pks ) == set :
    pks = list ( pks )
if len ( pks ) == 1 : # Optimization to not pipeline on 1 id
    return IRQueryableList ( [ self . get ( pks [ 0 ] , cascadeFetch = cascadeFetch ) ] , mdl = self . mdl )
conn = self . _get_connection ( )
pipeline = conn . pipeline ( )
for pk in pks :
    key = self . _get_key_for_id ( pk )
    pipeline . hgetall ( key )
res = pipeline . execute ( )
ret = IRQueryableList ( mdl = self . mdl )
i = 0
pksLen = len ( pks )
while i < pksLen :
    if res [ i ] is None :
        ret . append ( None )
        i += 1
        continue
    res [ i ] [ '_id' ] = pks [ i ]
    obj = self . _redisResultToObj ( res [ i ] )
    ret . append ( obj )
    i += 1
if cascadeFetch is True :
    for obj in ret :
        if not obj :
            continue
        self . _doCascadeFetch ( obj )
return ret


signature:
def getOnlyFields ( self , pk , fields , cascadeFetch = False ) :
    '''getOnlyFields - Gets only certain fields from a paticular primary key . For working on entire filter set , see allOnlyFields
 @ param pk < int > - Primary Key
 @ param fields list < str > - List of fields
 @ param cascadeFetch < bool > Default False , If True , all Foreign objects associated with this model
 will be fetched immediately . If False , foreign objects will be fetched on - access .
 return - Partial objects with only fields applied'''

implementation:
conn = self . _get_connection ( )
key = self . _get_key_for_id ( pk )
res = conn . hmget ( key , fields )
if type ( res ) != list or not len ( res ) :
    return None
objDict = { }
numFields = len ( fields )
i = 0
anyNotNone = False
while i < numFields :
    objDict [ fields [ i ] ] = res [ i ]
    if res [ i ] != None :
        anyNotNone = True
    i += 1
if anyNotNone is False :
    return None
objDict [ '_id' ] = pk
ret = self . _redisResultToObj ( objDict )
if cascadeFetch is True :
    self . _doCascadeFetch ( ret )
return ret


signature:
def getMultipleOnlyFields ( self , pks , fields , cascadeFetch = False ) :
    '''getMultipleOnlyFields - Gets only certain fields from a list of primary keys . For working on entire filter set , see allOnlyFields
 @ param pks list < str > - Primary Keys
 @ param fields list < str > - List of fields
 @ param cascadeFetch < bool > Default False , If True , all Foreign objects associated with this model
 will be fetched immediately . If False , foreign objects will be fetched on - access .
 return - List of partial objects with only fields applied'''

implementation:
if type ( pks ) == set :
    pks = list ( pks )
if len ( pks ) == 1 :
    return IRQueryableList ( [ self . getOnlyFields ( pks [ 0 ] , fields , cascadeFetch = cascadeFetch ) ] , mdl = self . mdl )
conn = self . _get_connection ( )
pipeline = conn . pipeline ( )
for pk in pks :
    key = self . _get_key_for_id ( pk )
    pipeline . hmget ( key , fields )
res = pipeline . execute ( )
ret = IRQueryableList ( mdl = self . mdl )
pksLen = len ( pks )
i = 0
numFields = len ( fields )
while i < pksLen :
    objDict = { }
    anyNotNone = False
    thisRes = res [ i ]
    if thisRes is None or type ( thisRes ) != list :
        ret . append ( None )
        i += 1
        continue
    j = 0
    while j < numFields :
        objDict [ fields [ j ] ] = thisRes [ j ]
        if thisRes [ j ] != None :
            anyNotNone = True
        j += 1
    if anyNotNone is False :
        ret . append ( None )
        i += 1
        continue
    objDict [ '_id' ] = pks [ i ]
    obj = self . _redisResultToObj ( objDict )
    ret . append ( obj )
    i += 1
if cascadeFetch is True :
    for obj in ret :
        self . _doCascadeFetch ( obj )
return ret


signature:
def reindex ( self ) :
    '''reindex - Reindexes the objects matching current filterset . Use this if you add / remove a field to INDEXED _ FIELDS .
 NOTE - This will NOT remove entries from the old index if you change index type , or change decimalPlaces on a
 IRFixedPointField . To correct these indexes , you ' ll need to run :
 Model . reset ( Model . objects . all ( ) )
 If you change the value of " hashIndex " on a field , you need to call # compat _ convertHashedIndexes instead .'''

implementation:
objs = self . all ( )
saver = IndexedRedisSave ( self . mdl )
saver . reindex ( objs )


signature:
def compat_convertHashedIndexes ( self , fetchAll = True ) :
    '''compat _ convertHashedIndexes - Reindex fields , used for when you change the propery " hashIndex " on one or more fields .
 For each field , this will delete both the hash and unhashed keys to an object ,
 and then save a hashed or unhashed value , depending on that field ' s value for " hashIndex " .
 For an IndexedRedisModel class named " MyModel " , call as " MyModel . objects . compat _ convertHashedIndexes ( ) "
 NOTE : This works one object at a time ( regardless of # fetchAll ) , so that an unhashable object does not trash all data .
 This method is intended to be used while your application is offline ,
 as it doesn ' t make sense to be changing your model while applications are actively using it .
 @ param fetchAll < bool > , Default True - If True , all objects will be fetched first , then converted .
 This is generally what you want to do , as it is more efficient . If you are memory contrainted ,
 you can set this to " False " , and it will fetch one object at a time , convert it , and save it back .'''

implementation:
saver = IndexedRedisSave ( self . mdl )
if fetchAll is True :
    objs = self . all ( )
    saver . compat_convertHashedIndexes ( objs )
else :
    didWarnOnce = False
    pks = self . getPrimaryKeys ( )
    for pk in pks :
        obj = self . get ( pk )
        if not obj :
            if didWarnOnce is False :
                sys . stderr . write ( 'WARNING(once)! An object (type=%s , pk=%d) disappered while ' 'running compat_convertHashedIndexes! This probably means an application ' 'is using the model while converting indexes. This is a very BAD IDEA (tm).' )
                didWarnOnce = True
            continue
        saver . compat_convertHashedIndexes ( [ obj ] )