signature:
def parse ( self , gff_file , strict = False ) :
    """Parse the gff file into the following data structures :
 * lines ( list of line _ data ( dict ) )
 - line _ index ( int ) : the index in lines
 - line _ raw ( str )
 - line _ type ( str in [ ' feature ' , ' directive ' , ' comment ' , ' blank ' , ' unknown ' ] )
 - line _ errors ( list of str ) : a list of error messages
 - line _ status ( str in [ ' normal ' , ' modified ' , ' removed ' ] )
 - parents ( list of feature ( list of line _ data ( dict ) ) ) : may have multiple parents
 - children ( list of line _ data ( dict ) )
 - extra fields depending on line _ type
 * directive
 - directive ( str in [ ' # # gff - version ' , ' # # sequence - region ' , ' # # feature - ontology ' , ' # # attribute - ontology ' , ' # # source - ontology ' , ' # # species ' , ' # # genome - build ' , ' # # # ' , ' # # FASTA ' ] )
 - extra fields depending on directive
 * feature
 - seqid ( str ) : must escape any characters not in the set [ a - zA - Z0-9 . : ^ * $ @ ! + _ ? - | ] using RFC 3986 Percent - Encoding
 - source ( str )
 - type ( str in so _ types )
 - start ( int )
 - end ( int )
 - score ( float )
 - strand ( str in [ ' + ' , ' - ' , ' . ' , ' ? ' ] )
 - phase ( int in [ 0 , 1 , 2 ] )
 - attributes ( dict of tag ( str ) to value )
 - ID ( str )
 - Name ( str )
 - Alias ( list of str ) : multi value
 - Parent ( list of str ) : multi value
 - Target ( dict )
 - target _ id ( str )
 - start ( int )
 - end ( int )
 - strand ( str in [ ' + ' , ' - ' , ' ' ] )
 - Gap ( str ) : CIGAR format
 - Derives _ from ( str )
 - Note ( list of str ) : multi value
 - Dbxref ( list of str ) : multi value
 - Ontology _ term ( list of str ) : multi value
 - Is _ circular ( str in [ ' true ' ] )
 * fasta _ dict ( dict of id ( str ) to sequence _ item ( dict ) )
 - id ( str )
 - header ( str )
 - seq ( str )
 - line _ length ( int )
 * features ( dict of feature _ id ( str in line _ data [ ' attributes ' ] [ ' ID ' ] ) to feature ( list of line _ data ( dict ) ) )
 A feature is a list of line _ data ( dict ) , since all lines that share an ID collectively represent a single feature .
 During serialization , line _ data ( dict ) references should be converted into line _ index ( int )
 : param gff _ file : a string path or file object
 : param strict : when true , throw exception on syntax and format errors . when false , use best effort to finish parsing while logging errors"""

implementation:
valid_strand = set ( ( '+' , '-' , '.' , '?' ) )
valid_phase = set ( ( 0 , 1 , 2 ) )
multi_value_attributes = set ( ( 'Parent' , 'Alias' , 'Note' , 'Dbxref' , 'Ontology_term' ) )
valid_attribute_target_strand = set ( ( '+' , '-' , '' ) )
reserved_attributes = set ( ( 'ID' , 'Name' , 'Alias' , 'Parent' , 'Target' , 'Gap' , 'Derives_from' , 'Note' , 'Dbxref' , 'Ontology_term' , 'Is_circular' ) )
# illegal character check
# Literal use of tab , newline , carriage return , the percent ( % ) sign , and control characters must be encoded using RFC 3986 Percent - Encoding ; no other characters may be encoded .
# control characters : \ x00 - \ x1f \ x7f this includes tab ( \ x09 ) , newline ( \ x0a ) , carriage return ( \ x0d )
# seqid may contain any characters , but must escape any characters not in the set [ a - zA - Z0-9 . : ^ * $ @ ! + _ ? - | ]
# URL escaping rules are used for tags or values containing the following characters : " , = ; " .
# > > > timeit ( " unescaped _ seqid ( ' Un . 7589 ' ) " , " import re ; unescaped _ seqid = re . compile ( r ' [ ^ a - zA - Z0-9 . : ^ * $ @ ! + _ ? | % - ] | % ( ? ! [ 0-9a - fA - F ] { 2 } ) ' ) . search " )
# 0.4128372745785036
# > > > timeit ( " unescaped _ seqid2 ( ' Un . 7589 ' ) " , " import re ; unescaped _ seqid2 = re . compile ( r ' ^ ( [ a - zA - Z0-9 . : ^ * $ @ ! + _ ? | - ] | % [ 0-9a - fA - F ] { 2 } ) + $ ' ) . search " )
# 0.9012313532265175
unescaped_seqid = re . compile ( r'[^a-zA-Z0-9.:^*$@!+_?|%-]|%(?![0-9a-fA-F]{2})' ) . search
unescaped_field = re . compile ( r'[\x00-\x1f\x7f]|%(?![0-9a-fA-F]{2})' ) . search
gff_fp = gff_file
if isinstance ( gff_file , str ) :
    gff_fp = open ( gff_file , 'r' )
lines = [ ]
current_line_num = 1
# line numbers start at 1
features = defaultdict ( list )
# key = the unresolved id , value = a list of line _ data ( dict )
unresolved_parents = defaultdict ( list )
for line_raw in gff_fp :
    line_data = { 'line_index' : current_line_num - 1 , 'line_raw' : line_raw , 'line_status' : 'normal' , 'parents' : [ ] , 'children' : [ ] , 'line_type' : '' , 'directive' : '' , 'line_errors' : [ ] , 'type' : '' , }
    line_strip = line_raw . strip ( )
    if line_strip != line_raw [ : len ( line_strip ) ] :
        self . add_line_error ( line_data , { 'message' : 'White chars not allowed at the start of a line' , 'error_type' : 'FORMAT' , 'location' : '' } )
    if current_line_num == 1 and not line_strip . startswith ( '##gff-version' ) :
        self . add_line_error ( line_data , { 'message' : '"##gff-version" missing from the first line' , 'error_type' : 'FORMAT' , 'location' : '' } )
    if len ( line_strip ) == 0 :
        line_data [ 'line_type' ] = 'blank'
        continue
    if line_strip . startswith ( '##' ) :
        line_data [ 'line_type' ] = 'directive'
        if line_strip . startswith ( '##sequence-region' ) : # # # sequence - region seqid start end
        # This element is optional , but strongly encouraged because it allows parsers to perform bounds checking on features .
        # only one # # sequence - region directive may be given for any given seqid
        # all features on that landmark feature ( having that seqid ) must be contained within the range defined by that # # sequence - region diretive . An exception to this rule is allowed when a landmark feature is marked with the Is _ circular attribute .
            line_data [ 'directive' ] = '##sequence-region'
            tokens = list ( line_strip . split ( ) [ 1 : ] )
            if len ( tokens ) != 3 :
                self . add_line_error ( line_data , { 'message' : 'Expecting 3 fields, got %d: %s' % ( len ( tokens ) - 1 , repr ( tokens [ 1 : ] ) ) , 'error_type' : 'FORMAT' , 'location' : '' } )
            if len ( tokens ) > 0 :
                line_data [ 'seqid' ] = tokens [ 0 ]
                # check for duplicate # # sequence - region seqid
                if [ True for d in lines if ( 'directive' in d and d [ 'directive' ] == '##sequence-region' and 'seqid' in d and d [ 'seqid' ] == line_data [ 'seqid' ] ) ] :
                    self . add_line_error ( line_data , { 'message' : '##sequence-region seqid: "%s" may only appear once' % line_data [ 'seqid' ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                try :
                    all_good = True
                    try :
                        line_data [ 'start' ] = int ( tokens [ 1 ] )
                        if line_data [ 'start' ] < 1 :
                            self . add_line_error ( line_data , { 'message' : 'Start is not a valid 1-based integer coordinate: "%s"' % tokens [ 1 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                    except ValueError :
                        all_good = False
                        self . add_line_error ( line_data , { 'message' : 'Start is not a valid integer: "%s"' % tokens [ 1 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                        line_data [ 'start' ] = tokens [ 1 ]
                    try :
                        line_data [ 'end' ] = int ( tokens [ 2 ] )
                        if line_data [ 'end' ] < 1 :
                            self . add_line_error ( line_data , { 'message' : 'End is not a valid 1-based integer coordinate: "%s"' % tokens [ 2 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                    except ValueError :
                        all_good = False
                        self . add_line_error ( line_data , { 'message' : 'End is not a valid integer: "%s"' % tokens [ 2 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                        line_data [ 'start' ] = tokens [ 2 ]
                    # if all _ good then both start and end are int , so we can check if start is not less than or equal to end
                    if all_good and line_data [ 'start' ] > line_data [ 'end' ] :
                        self . add_line_error ( line_data , { 'message' : 'Start is not less than or equal to end' , 'error_type' : 'FORMAT' , 'location' : '' } )
                except IndexError :
                    pass
        elif line_strip . startswith ( '##gff-version' ) : # The GFF version , always 3 in this specification must be present , must be the topmost line of the file and may only appear once in the file .
            line_data [ 'directive' ] = '##gff-version'
            # check if it appeared before
            if [ True for d in lines if ( 'directive' in d and d [ 'directive' ] == '##gff-version' ) ] :
                self . add_line_error ( line_data , { 'message' : '##gff-version missing from the first line' , 'error_type' : 'FORMAT' , 'location' : '' } )
            tokens = list ( line_strip . split ( ) [ 1 : ] )
            if len ( tokens ) != 1 :
                self . add_line_error ( line_data , { 'message' : 'Expecting 1 field, got %d: %s' % ( len ( tokens ) - 1 , repr ( tokens [ 1 : ] ) ) , 'error_type' : 'FORMAT' , 'location' : '' } )
            if len ( tokens ) > 0 :
                try :
                    line_data [ 'version' ] = int ( tokens [ 0 ] )
                    if line_data [ 'version' ] != 3 :
                        self . add_line_error ( line_data , { 'message' : 'Version is not "3": "%s"' % tokens [ 0 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                except ValueError :
                    self . add_line_error ( line_data , { 'message' : 'Version is not a valid integer: "%s"' % tokens [ 0 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                    line_data [ 'version' ] = tokens [ 0 ]
        elif line_strip . startswith ( '###' ) : # This directive ( three # signs in a row ) indicates that all forward references to feature IDs that have been seen to this point have been resolved .
            line_data [ 'directive' ] = '###'
        elif line_strip . startswith ( '##FASTA' ) : # This notation indicates that the annotation portion of the file is at an end and that the
        # remainder of the file contains one or more sequences ( nucleotide or protein ) in FASTA format .
            line_data [ 'directive' ] = '##FASTA'
            self . logger . info ( 'Reading embedded ##FASTA sequence' )
            self . fasta_embedded , count = fasta_file_to_dict ( gff_fp )
            self . logger . info ( '%d sequences read' % len ( self . fasta_embedded ) )
        elif line_strip . startswith ( '##feature-ontology' ) : # # # feature - ontology URI
        # This directive indicates that the GFF3 file uses the ontology of feature types located at the indicated URI or URL .
            line_data [ 'directive' ] = '##feature-ontology'
            tokens = list ( line_strip . split ( ) [ 1 : ] )
            if len ( tokens ) != 1 :
                self . add_line_error ( line_data , { 'message' : 'Expecting 1 field, got %d: %s' % ( len ( tokens ) - 1 , repr ( tokens [ 1 : ] ) ) , 'error_type' : 'FORMAT' , 'location' : '' } )
            if len ( tokens ) > 0 :
                line_data [ 'URI' ] = tokens [ 0 ]
        elif line_strip . startswith ( '##attribute-ontology' ) : # # # attribute - ontology URI
        # This directive indicates that the GFF3 uses the ontology of attribute names located at the indicated URI or URL .
            line_data [ 'directive' ] = '##attribute-ontology'
            tokens = list ( line_strip . split ( ) [ 1 : ] )
            if len ( tokens ) != 1 :
                self . add_line_error ( line_data , { 'message' : 'Expecting 1 field, got %d: %s' % ( len ( tokens ) - 1 , repr ( tokens [ 1 : ] ) ) , 'error_type' : 'FORMAT' , 'location' : '' } )
            if len ( tokens ) > 0 :
                line_data [ 'URI' ] = tokens [ 0 ]
        elif line_strip . startswith ( '##source-ontology' ) : # # # source - ontology URI
        # This directive indicates that the GFF3 uses the ontology of source names located at the indicated URI or URL .
            line_data [ 'directive' ] = '##source-ontology'
            tokens = list ( line_strip . split ( ) [ 1 : ] )
            if len ( tokens ) != 1 :
                self . add_line_error ( line_data , { 'message' : 'Expecting 1 field, got %d: %s' % ( len ( tokens ) - 1 , repr ( tokens [ 1 : ] ) ) , 'error_type' : 'FORMAT' , 'location' : '' } )
            if len ( tokens ) > 0 :
                line_data [ 'URI' ] = tokens [ 0 ]
        elif line_strip . startswith ( '##species' ) : # # # species NCBI _ Taxonomy _ URI
        # This directive indicates the species that the annotations apply to .
            line_data [ 'directive' ] = '##species'
            tokens = list ( line_strip . split ( ) [ 1 : ] )
            if len ( tokens ) != 1 :
                self . add_line_error ( line_data , { 'message' : 'Expecting 1 field, got %d: %s' % ( len ( tokens ) - 1 , repr ( tokens [ 1 : ] ) ) , 'error_type' : 'FORMAT' , 'location' : '' } )
            if len ( tokens ) > 0 :
                line_data [ 'NCBI_Taxonomy_URI' ] = tokens [ 0 ]
        elif line_strip . startswith ( '##genome-build' ) : # # # genome - build source buildName
        # The genome assembly build name used for the coordinates given in the file .
            line_data [ 'directive' ] = '##genome-build'
            tokens = list ( line_strip . split ( ) [ 1 : ] )
            if len ( tokens ) != 2 :
                self . add_line_error ( line_data , { 'message' : 'Expecting 2 fields, got %d: %s' % ( len ( tokens ) - 1 , repr ( tokens [ 1 : ] ) ) , 'error_type' : 'FORMAT' , 'location' : '' } )
            if len ( tokens ) > 0 :
                line_data [ 'source' ] = tokens [ 0 ]
                try :
                    line_data [ 'buildName' ] = tokens [ 1 ]
                except IndexError :
                    pass
        else :
            self . add_line_error ( line_data , { 'message' : 'Unknown directive' , 'error_type' : 'FORMAT' , 'location' : '' } )
            tokens = list ( line_strip . split ( ) )
            line_data [ 'directive' ] = tokens [ 0 ]
    elif line_strip . startswith ( '#' ) :
        line_data [ 'line_type' ] = 'comment'
    else : # line _ type may be a feature or unknown
        line_data [ 'line_type' ] = 'feature'
        tokens = list ( map ( str . strip , line_raw . split ( '\t' ) ) )
        if len ( tokens ) != 9 :
            self . add_line_error ( line_data , { 'message' : 'Features should contain 9 fields, got %d: %s' % ( len ( tokens ) - 1 , repr ( tokens [ 1 : ] ) ) , 'error_type' : 'FORMAT' , 'location' : '' } )
        for i , t in enumerate ( tokens ) :
            if not t :
                self . add_line_error ( line_data , { 'message' : 'Empty field: %d, must have a "."' % ( i + 1 ) , 'error_type' : 'FORMAT' , 'location' : '' } )
        try :
            line_data [ 'seqid' ] = tokens [ 0 ]
            if unescaped_seqid ( tokens [ 0 ] ) :
                self . add_line_error ( line_data , { 'message' : 'Seqid must escape any characters not in the set [a-zA-Z0-9.:^*$@!+_?-|]: "%s"' % tokens [ 0 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
            line_data [ 'source' ] = tokens [ 1 ]
            if unescaped_field ( tokens [ 1 ] ) :
                self . add_line_error ( line_data , { 'message' : 'Source must escape the percent (%%) sign and any control characters: "%s"' % tokens [ 1 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
            line_data [ 'type' ] = tokens [ 2 ]
            if unescaped_field ( tokens [ 2 ] ) :
                self . add_line_error ( line_data , { 'message' : 'Type must escape the percent (%%) sign and any control characters: "%s"' % tokens [ 2 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
            all_good = True
            try :
                line_data [ 'start' ] = int ( tokens [ 3 ] )
                if line_data [ 'start' ] < 1 :
                    self . add_line_error ( line_data , { 'message' : 'Start is not a valid 1-based integer coordinate: "%s"' % tokens [ 3 ] , 'error_type' : 'FORMAT' , 'location' : 'start' } )
            except ValueError :
                all_good = False
                line_data [ 'start' ] = tokens [ 3 ]
                if line_data [ 'start' ] != '.' :
                    self . add_line_error ( line_data , { 'message' : 'Start is not a valid integer: "%s"' % line_data [ 'start' ] , 'error_type' : 'FORMAT' , 'location' : 'start' } )
            try :
                line_data [ 'end' ] = int ( tokens [ 4 ] )
                if line_data [ 'end' ] < 1 :
                    self . add_line_error ( line_data , { 'message' : 'End is not a valid 1-based integer coordinate: "%s"' % tokens [ 4 ] , 'error_type' : 'FORMAT' , 'location' : 'end' } )
            except ValueError :
                all_good = False
                line_data [ 'end' ] = tokens [ 4 ]
                if line_data [ 'end' ] != '.' :
                    self . add_line_error ( line_data , { 'message' : 'End is not a valid integer: "%s"' % line_data [ 'end' ] , 'error_type' : 'FORMAT' , 'location' : 'end' } )
            # if all _ good then both start and end are int , so we can check if start is not less than or equal to end
            if all_good and line_data [ 'start' ] > line_data [ 'end' ] :
                self . add_line_error ( line_data , { 'message' : 'Start is not less than or equal to end' , 'error_type' : 'FORMAT' , 'location' : 'start,end' } )
            try :
                line_data [ 'score' ] = float ( tokens [ 5 ] )
            except ValueError :
                line_data [ 'score' ] = tokens [ 5 ]
                if line_data [ 'score' ] != '.' :
                    self . add_line_error ( line_data , { 'message' : 'Score is not a valid floating point number: "%s"' % line_data [ 'score' ] , 'error_type' : 'FORMAT' , 'location' : '' } )
            line_data [ 'strand' ] = tokens [ 6 ]
            if line_data [ 'strand' ] not in valid_strand : # set ( [ ' + ' , ' - ' , ' . ' , ' ? ' ] )
                self . add_line_error ( line_data , { 'message' : 'Strand has illegal characters: "%s"' % tokens [ 6 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
            try :
                line_data [ 'phase' ] = int ( tokens [ 7 ] )
                if line_data [ 'phase' ] not in valid_phase : # set ( [ 0 , 1 , 2 ] )
                    self . add_line_error ( line_data , { 'message' : 'Phase is not 0, 1, or 2: "%s"' % tokens [ 7 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
            except ValueError :
                line_data [ 'phase' ] = tokens [ 7 ]
                if line_data [ 'phase' ] != '.' :
                    self . add_line_error ( line_data , { 'message' : 'Phase is not a valid integer: "%s"' % line_data [ 'phase' ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                elif line_data [ 'type' ] == 'CDS' :
                    self . add_line_error ( line_data , { 'message' : 'Phase is required for all CDS features' , 'error_type' : 'FORMAT' , 'location' : '' } )
            # parse attributes , ex : ID = exon00003 ; Parent = mRNA00001 , mRNA00003 ; Name = EXON . 1
            # URL escaping rules are used for tags or values containing the following characters : " , = ; " . Spaces are allowed in this field , but tabs must be replaced with the % 09 URL escape .
            # Note that attribute names are case sensitive . " Parent " is not the same as " parent " .
            # All attributes that begin with an uppercase letter are reserved for later use . Attributes that begin with a lowercase letter can be used freely by applications .
            if unescaped_field ( tokens [ 8 ] ) :
                self . add_line_error ( line_data , { 'message' : 'Attributes must escape the percent (%) sign and any control characters' , 'error_type' : 'FORMAT' , 'location' : '' } )
            attribute_tokens = tuple ( tuple ( t for t in a . split ( '=' ) ) for a in tokens [ 8 ] . split ( ';' ) if a )
            line_data [ 'attributes' ] = { }
            if len ( attribute_tokens ) == 1 and len ( attribute_tokens [ 0 ] ) == 1 and attribute_tokens [ 0 ] [ 0 ] == '.' :
                pass
                # no attributes
            else :
                for a in attribute_tokens :
                    if len ( a ) != 2 :
                        self . add_line_error ( line_data , { 'message' : 'Attributes must contain one and only one equal (=) sign: "%s"' % ( '=' . join ( a ) ) , 'error_type' : 'FORMAT' , 'location' : '' } )
                    try :
                        tag , value = a
                    except ValueError :
                        tag , value = a [ 0 ] , ''
                    if not tag :
                        self . add_line_error ( line_data , { 'message' : 'Empty attribute tag: "%s"' % '=' . join ( a ) , 'error_type' : 'FORMAT' , 'location' : '' } )
                    if not value . strip ( ) :
                        self . add_line_error ( line_data , { 'message' : 'Empty attribute value: "%s"' % '=' . join ( a ) , 'error_type' : 'FORMAT' , 'location' : '' } , log_level = logging . WARNING )
                    if tag in line_data [ 'attributes' ] :
                        self . add_line_error ( line_data , { 'message' : 'Found multiple attribute tags: "%s"' % tag , 'error_type' : 'FORMAT' , 'location' : '' } )
                    if tag in multi_value_attributes : # set ( [ ' Parent ' , ' Alias ' , ' Note ' , ' Dbxref ' , ' Ontology _ term ' ] )
                        if value . find ( ', ' ) >= 0 :
                            self . add_line_error ( line_data , { 'message' : 'Found ", " in %s attribute, possible unescaped ",": "%s"' % ( tag , value ) , 'error_type' : 'FORMAT' , 'location' : '' } , log_level = logging . WARNING )
                        # In addition to Parent , the Alias , Note , Dbxref and Ontology _ term attributes can have multiple values .
                        if tag in line_data [ 'attributes' ] : # if this tag has been seen before
                            if tag == 'Note' : # don ' t check for duplicate notes
                                line_data [ 'attributes' ] [ tag ] . extend ( value . split ( ',' ) )
                            else : # only add non duplicate values
                                line_data [ 'attributes' ] [ tag ] . extend ( [ s for s in value . split ( ',' ) if s not in line_data [ 'attributes' ] [ tag ] ] )
                        else :
                            line_data [ 'attributes' ] [ tag ] = value . split ( ',' )
                        # check for duplicate values
                        if tag != 'Note' and len ( line_data [ 'attributes' ] [ tag ] ) != len ( set ( line_data [ 'attributes' ] [ tag ] ) ) :
                            count_values = [ ( len ( list ( group ) ) , key ) for key , group in groupby ( sorted ( line_data [ 'attributes' ] [ tag ] ) ) ]
                            self . add_line_error ( line_data , { 'message' : '%s attribute has identical values (count, value): %s' % ( tag , ', ' . join ( [ '(%d, %s)' % ( c , v ) for c , v in count_values if c > 1 ] ) ) , 'error_type' : 'FORMAT' , 'location' : '' } )
                            # remove duplicate
                            line_data [ 'attributes' ] [ tag ] = list ( set ( line_data [ 'attributes' ] [ tag ] ) )
                        if tag == 'Parent' :
                            for feature_id in line_data [ 'attributes' ] [ 'Parent' ] :
                                try :
                                    line_data [ 'parents' ] . append ( features [ feature_id ] )
                                    for ld in features [ feature_id ] : # no need to check if line _ data in ld [ ' children ' ] , because it is impossible , each ld maps to only one feature _ id , so the ld we get are all different
                                        ld [ 'children' ] . append ( line_data )
                                except KeyError : # features [ id ]
                                    self . add_line_error ( line_data , { 'message' : '%s attribute has unresolved forward reference: %s' % ( tag , feature_id ) , 'error_type' : 'FORMAT' , 'location' : '' } )
                                    unresolved_parents [ feature_id ] . append ( line_data )
                    elif tag == 'Target' :
                        if value . find ( ',' ) >= 0 :
                            self . add_line_error ( line_data , { 'message' : 'Value of %s attribute contains unescaped ",": "%s"' % ( tag , value ) , 'error_type' : 'FORMAT' , 'location' : '' } )
                        target_tokens = value . split ( ' ' )
                        if len ( target_tokens ) < 3 or len ( target_tokens ) > 4 :
                            self . add_line_error ( line_data , { 'message' : 'Target attribute should have 3 or 4 values, got %d: %s' % ( len ( target_tokens ) , repr ( tokens ) ) , 'error_type' : 'FORMAT' , 'location' : '' } )
                        line_data [ 'attributes' ] [ tag ] = { }
                        try :
                            line_data [ 'attributes' ] [ tag ] [ 'target_id' ] = target_tokens [ 0 ]
                            all_good = True
                            try :
                                line_data [ 'attributes' ] [ tag ] [ 'start' ] = int ( target_tokens [ 1 ] )
                                if line_data [ 'attributes' ] [ tag ] [ 'start' ] < 1 :
                                    self . add_line_error ( line_data , { 'message' : 'Start value of Target attribute is not a valid 1-based integer coordinate: "%s"' % target_tokens [ 1 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                            except ValueError :
                                all_good = False
                                line_data [ 'attributes' ] [ tag ] [ 'start' ] = target_tokens [ 1 ]
                                self . add_line_error ( line_data , { 'message' : 'Start value of Target attribute is not a valid integer: "%s"' % line_data [ 'attributes' ] [ tag ] [ 'start' ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                            try :
                                line_data [ 'attributes' ] [ tag ] [ 'end' ] = int ( target_tokens [ 2 ] )
                                if line_data [ 'attributes' ] [ tag ] [ 'end' ] < 1 :
                                    self . add_line_error ( line_data , { 'message' : 'End value of Target attribute is not a valid 1-based integer coordinate: "%s"' % target_tokens [ 2 ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                            except ValueError :
                                all_good = False
                                line_data [ 'attributes' ] [ tag ] [ 'end' ] = target_tokens [ 2 ]
                                self . add_line_error ( line_data , { 'message' : 'End value of Target attribute is not a valid integer: "%s"' % line_data [ 'attributes' ] [ tag ] [ 'end' ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                            # if all _ good then both start and end are int , so we can check if start is not less than or equal to end
                            if all_good and line_data [ 'attributes' ] [ tag ] [ 'start' ] > line_data [ 'attributes' ] [ tag ] [ 'end' ] :
                                self . add_line_error ( line_data , { 'message' : 'Start is not less than or equal to end' , 'error_type' : 'FORMAT' , 'location' : '' } )
                            line_data [ 'attributes' ] [ tag ] [ 'strand' ] = target_tokens [ 3 ]
                            if line_data [ 'attributes' ] [ tag ] [ 'strand' ] not in valid_attribute_target_strand : # set ( [ ' + ' , ' - ' , ' ' ] )
                                self . add_line_error ( line_data , { 'message' : 'Strand value of Target attribute has illegal characters: "%s"' % line_data [ 'attributes' ] [ tag ] [ 'strand' ] , 'error_type' : 'FORMAT' , 'location' : '' } )
                        except IndexError :
                            pass
                    else :
                        if value . find ( ',' ) >= 0 :
                            self . add_line_error ( line_data , { 'message' : 'Value of %s attribute contains unescaped ",": "%s"' % ( tag , value ) , 'error_type' : 'FORMAT' , 'location' : '' } )
                        line_data [ 'attributes' ] [ tag ] = value
                        if tag == 'Is_circular' and value != 'true' :
                            self . add_line_error ( line_data , { 'message' : 'Value of Is_circular attribute is not "true": "%s"' % value , 'error_type' : 'FORMAT' , 'location' : '' } )
                        elif tag [ : 1 ] . isupper ( ) and tag not in reserved_attributes : # { ' ID ' , ' Name ' , ' Alias ' , ' Parent ' , ' Target ' , ' Gap ' , ' Derives _ from ' , ' Note ' , ' Dbxref ' , ' Ontology _ term ' , ' Is _ circular ' }
                            self . add_line_error ( line_data , { 'message' : 'Unknown reserved (uppercase) attribute: "%s"' % tag , 'error_type' : 'FORMAT' , 'location' : '' } )
                        elif tag == 'ID' : # check for duplicate ID in non - adjacent lines
                            if value in features and lines [ - 1 ] [ 'attributes' ] [ tag ] != value :
                                self . add_line_error ( line_data , { 'message' : 'Duplicate ID: "%s" in non-adjacent lines: %s' % ( value , ',' . join ( [ str ( f [ 'line_index' ] + 1 ) for f in features [ value ] ] ) ) , 'error_type' : 'FORMAT' , 'location' : '' } , log_level = logging . WARNING )
                            features [ value ] . append ( line_data )
        except IndexError :
            pass
    current_line_num += 1
    lines . append ( line_data )
if isinstance ( gff_file , str ) :
    gff_fp . close ( )
# global look up of unresolved parents
for feature_id in unresolved_parents :
    if feature_id in features :
        for line in unresolved_parents [ feature_id ] :
            self . add_line_error ( line , { 'message' : 'Unresolved forward reference: "%s", found defined in lines: %s' % ( feature_id , ',' . join ( [ str ( ld [ 'line_index' ] + 1 ) for ld in features [ feature_id ] ] ) ) , 'error_type' : 'FORMAT' , 'location' : '' } )
self . lines = lines
self . features = features
return 1


signature:
def descendants ( self , line_data ) :
    """BFS graph algorithm
 : param line _ data : line _ data ( dict ) with line _ data [ ' line _ index ' ] or line _ index ( int )
 : return : list of line _ data ( dict )"""

implementation:
# get start node
try :
    start = line_data [ 'line_index' ]
except TypeError :
    start = self . lines [ line_data ] [ 'line_index' ]
visited_set , visited_list , queue = set ( ) , [ ] , [ start ]
while queue :
    node = queue . pop ( 0 )
    if node not in visited_set :
        visited_set . add ( node )
        visited_list . append ( self . lines [ node ] )
        queue . extend ( [ ld [ 'line_index' ] for ld in self . lines [ node ] [ 'children' ] if ld [ 'line_index' ] not in visited_set ] )
return visited_list [ 1 : ]


signature:
def adopt ( self , old_parent , new_parent ) :
    """Transfer children from old _ parent to new _ parent
 : param old _ parent : feature _ id ( str ) or line _ index ( int ) or line _ data ( dict ) or feature
 : param new _ parent : feature _ id ( str ) or line _ index ( int ) or line _ data ( dict )
 : return : List of children transferred"""

implementation:
try : # assume line _ data ( dict )
    old_id = old_parent [ 'attributes' ] [ 'ID' ]
except TypeError :
    try : # assume line _ index ( int )
        old_id = self . lines [ old_parent ] [ 'attributes' ] [ 'ID' ]
    except TypeError : # assume feature _ id ( str )
        old_id = old_parent
old_feature = self . features [ old_id ]
old_indexes = [ ld [ 'line_index' ] for ld in old_feature ]
try : # assume line _ data ( dict )
    new_id = new_parent [ 'attributes' ] [ 'ID' ]
except TypeError :
    try : # assume line _ index ( int )
        new_id = self . lines [ new_parent ] [ 'attributes' ] [ 'ID' ]
    except TypeError : # assume feature _ id ( str )
        new_id = new_parent
new_feature = self . features [ new_id ]
new_indexes = [ ld [ 'line_index' ] for ld in new_feature ]
# build a list of children to be moved
# add the child to the new parent ' s children list if its not already there
# update the child ' s parent list and parent attribute
# finally remove the old parent ' s children list
children = old_feature [ 0 ] [ 'children' ]
new_parent_children_set = set ( [ ld [ 'line_index' ] for ld in new_feature [ 0 ] [ 'children' ] ] )
for child in children :
    if child [ 'line_index' ] not in new_parent_children_set :
        new_parent_children_set . add ( child [ 'line_index' ] )
        for new_ld in new_feature :
            new_ld [ 'children' ] . append ( child )
        child [ 'parents' ] . append ( new_feature )
        child [ 'attributes' ] [ 'Parent' ] . append ( new_id )
    # remove multiple , list . remove ( ) only removes 1
    child [ 'parents' ] = [ f for f in child [ 'parents' ] if f [ 0 ] [ 'attributes' ] [ 'ID' ] != old_id ]
    child [ 'attributes' ] [ 'Parent' ] = [ d for d in child [ 'attributes' ] [ 'Parent' ] if d != old_id ]
for old_ld in old_feature :
    old_ld [ 'children' ] = [ ]
return children


signature:
def remove ( self , line_data , root_type = None ) :
    """Marks line _ data and all of its associated feature ' s ' line _ status ' as ' removed ' , does not actually remove the line _ data from the data structure .
 The write function checks the ' line _ status ' when writing the gff file .
 Find the root parent of line _ data of type root _ type , remove all of its descendants .
 If the root parent has a parent with no children after the remove , remove the root parent ' s parent recursively .
 : param line _ data :
 : param root _ type :
 : return :"""

implementation:
roots = [ ld for ld in self . ancestors ( line_data ) if ( root_type and ld [ 'line_type' ] == root_type ) or ( not root_type and not ld [ 'parents' ] ) ] or [ line_data ]
for root in roots :
    root [ 'line_status' ] = 'removed'
    root_descendants = self . descendants ( root )
    for root_descendant in root_descendants :
        root_descendant [ 'line_status' ] = 'removed'
    root_ancestors = self . ancestors ( root )
    # BFS , so we will process closer ancestors first
    for root_ancestor in root_ancestors :
        if len ( [ ld for ld in root_ancestor [ 'children' ] if ld [ 'line_status' ] != 'removed' ] ) == 0 : # if all children of a root _ ancestor is removed
        # remove this root _ ancestor
            root_ancestor [ 'line_status' ] = 'removed'


signature:
def sequence ( self , line_data , child_type = None , reference = None ) :
    """Get the sequence of line _ data , according to the columns ' seqid ' , ' start ' , ' end ' , ' strand ' .
 Requires fasta reference .
 When used on ' mRNA ' type line _ data , child _ type can be used to specify which kind of sequence to return :
 * child _ type = None : pre - mRNA , returns the sequence of line _ data from start to end , reverse complement according to strand . ( default )
 * child _ type = ' exon ' : mature mRNA , concatenates the sequences of children type ' exon ' .
 * child _ type = ' CDS ' : coding sequence , concatenates the sequences of children type ' CDS ' . Use the helper
 function translate ( seq ) on the returned value to obtain the protein sequence .
 : param line _ data : line _ data ( dict ) with line _ data [ ' line _ index ' ] or line _ index ( int )
 : param child _ type : None or feature type ( string )
 : param reference : If None , will use self . fasta _ external or self . fasta _ embedded ( dict )
 : return : sequence ( string )"""

implementation:
# get start node
reference = reference or self . fasta_external or self . fasta_embedded
if not reference :
    raise Exception ( 'External or embedded fasta reference needed' )
try :
    line_index = line_data [ 'line_index' ]
except TypeError :
    line_index = self . lines [ line_data ] [ 'line_index' ]
ld = self . lines [ line_index ]
if ld [ 'type' ] != 'feature' :
    return None
seq = reference [ ld [ 'seqid' ] ] [ ld [ 'start' ] - 1 : ld [ 'end' ] ]
if ld [ 'strand' ] == '-' :
    seq = complement ( seq [ : : - 1 ] )
return seq


signature:
def abfIDfromFname ( fname ) :
    """given a filename , return the ABFs ID string ."""

implementation:
fname = os . path . abspath ( fname )
basename = os . path . basename ( fname )
return os . path . splitext ( basename ) [ 0 ]


signature:
def abfProtocol ( fname ) :
    """Determine the protocol used to record an ABF file"""

implementation:
f = open ( fname , 'rb' )
raw = f . read ( 30 * 1000 )
# it should be in the first 30k of the file
f . close ( )
raw = raw . decode ( "utf-8" , "ignore" )
raw = raw . split ( "Clampex" ) [ 1 ] . split ( ".pro" ) [ 0 ]
protocol = os . path . basename ( raw )
# the whole protocol filename
protocolID = protocol . split ( " " ) [ 0 ]
# just the first number
return protocolID


signature:
def headerHTML ( header , fname ) :
    """given the bytestring ABF header , make and launch HTML ."""

implementation:
html = ""
html += "
%s
" % ( fname )
html += pprint . pformat ( header , indent = 1 )
html = html . replace ( "\n" , '
' ) . replace ( " " , " " )
html = html . replace ( r"\x00" , "" )
html += ""
print ( "saving header file:" , fname )
f = open ( fname , 'w' )
f . write ( html )
f . close ( )
webbrowser . open ( fname )


signature:
def setsweep ( self , sweep = 0 , channel = 0 ) :
    """set the sweep and channel of an ABF . Both start at 0."""

implementation:
try :
    sweep = int ( sweep )
except :
    self . log . error ( "trying to set sweep to [%s]" , sweep )
    return
if sweep < 0 :
    sweep = self . sweeps - 1 - sweep
    # if negative , start from the end
sweep = max ( 0 , min ( sweep , self . sweeps - 1 ) )
# correct for out of range sweeps
if 'sweep' in dir ( self ) and self . sweep == sweep and self . derivative is False :
    self . log . debug ( "sweep %d already set" , sweep )
    return
# self . log . debug ( " loading sweep % d ( Ch % d ) " , sweep , channel )
self . channels = self . ABFblock . segments [ sweep ] . size [ "analogsignals" ]
if self . channels > 1 and sweep == 0 :
    self . log . info ( "WARNING: multichannel not yet supported!" )
    # TODO :
self . trace = self . ABFblock . segments [ sweep ] . analogsignals [ channel ]
self . sweep = sweep
# currently selected sweep
self . channel = channel
# currently selected channel
# sweep information
self . rate = int ( self . trace . sampling_rate )
# Hz
self . period = float ( 1 / self . rate )
# seconds ( inverse of sample rate )
self . pointsPerSec = int ( self . rate )
# for easy access
self . pointsPerMs = int ( self . rate / 1000.0 )
# for easy access
self . sweepSize = len ( self . trace )
# number of data points per sweep
self . sweepInterval = self . trace . duration . magnitude
# sweep interval ( seconds )
self . sweepLength = float ( self . trace . t_stop - self . trace . t_start )
# in seconds
self . length = self . sweepLength * self . sweeps
# length ( sec ) of total recording
self . lengthMinutes = self . length / 60.0
# length ( minutes ) of total recording
if str ( self . trace . dimensionality ) == 'pA' :
    self . units , self . units2 = "pA" , "clamp current (pA)"
    self . unitsD , self . unitsD2 = "pA/ms" , "current velocity (pA/ms)"
    self . protoUnits , self . protoUnits2 = "mV" , "command voltage (mV)"
elif str ( self . trace . dimensionality ) == 'mV' :
    self . units , self . units2 = "mV" , "membrane potential (mV)"
    self . unitsD , self . unitsD2 = "V/s" , "potential velocity (V/s)"
    self . protoUnits , self . protoUnits2 = "pA" , "command current (pA)"
else :
    self . units , self . units2 = "?" , "unknown units"
    self . unitsD , self . unitsD2 = "?" , "unknown units"
# sweep data
self . sweepY = self . trace . magnitude
# sweep data ( mV or pA )
self . sweepT = self . trace . times . magnitude
# actual sweep times ( sec )
self . sweepStart = float ( self . trace . t_start )
# time start of sweep ( sec )
self . sweepX2 = self . sweepT - self . trace . t_start . magnitude
# sweeps overlap
self . sweepX = self . sweepX2 + sweep * self . sweepInterval
# assume no gaps
if self . derivative :
    self . log . debug ( "taking derivative" )
    # self . sweepD = np . diff ( self . sweepY ) # take derivative
    self . sweepD = self . sweepY [ 1 : ] - self . sweepY [ : - 1 ]
    # better ?
    self . sweepD = np . insert ( self . sweepD , 0 , self . sweepD [ 0 ] )
    # add a point
    self . sweepD /= ( self . period * 1000 )
    # correct for sample rate
else :
    self . sweepD = [ 0 ]
# derivative is forced to be empty
# generate the protocol too
self . generate_protocol ( )


signature:
def setsweeps ( self ) :
    """iterate over every sweep"""

implementation:
for sweep in range ( self . sweeps ) :
    self . setsweep ( sweep )
    yield self . sweep


signature:
def comments_load ( self ) :
    """read the header and populate self with information about comments"""

implementation:
self . comment_times , self . comment_sweeps , self . comment_tags = [ ] , [ ] , [ ]
self . comments = 0
# will be > 0 if comments exist
self . comment_text = ""
try : # this used to work
    self . comment_tags = list ( self . ABFblock . segments [ 0 ] . eventarrays [ 0 ] . annotations [ 'comments' ] )
    self . comment_times = list ( self . ABFblock . segments [ 0 ] . eventarrays [ 0 ] . times / self . trace . itemsize )
    self . comment_sweeps = list ( self . comment_times )
except : # now this notation seems to work
    for events in self . ABFblock . segments [ 0 ] . events : # this should only happen once actually
        self . comment_tags = events . annotations [ 'comments' ] . tolist ( )
        self . comment_times = np . array ( events . times . magnitude / self . trace . itemsize )
        self . comment_sweeps = self . comment_times / self . sweepInterval
for i , c in enumerate ( self . comment_tags ) :
    self . comment_tags [ i ] = c . decode ( "utf-8" )


signature:
def generate_protocol ( self ) :
    """Recreate the command stimulus ( protocol ) for the current sweep .
 It ' s not stored point by point ( that ' s a waste of time and memory ! )
 Instead it ' s stored as a few ( x , y ) points which can be easily graphed .
 TODO : THIS
 for segment in abf . ABFreader . read _ protocol ( ) :
 for analogsignal in segment . analogsignals :
 print ( analogsignal )
 plt . plot ( analogsignal )
 plt . show ( )
 plt . close ( ' all ' )"""

implementation:
# TODO : elegantly read the protocol like this :
# abf . ABFreader . read _ protocol ( ) [ 0 ] . analogsignals ( ) [ sigNum ]
# TODO : right now this works only for the first channel
# correct for weird recording / protocol misalignment
# what is magic here ? 64 - bit data points ? # 1,000,000/64 = 15625 btw
self . offsetX = int ( self . sweepSize / 64 )
# if there ' s not a header , get out of here !
if not len ( self . header [ 'dictEpochInfoPerDAC' ] ) :
    self . log . debug ( "no protocol defined, so I'll make one" )
    self . protoX , self . protoY = [ 0 , self . sweepX [ - 1 ] ] , [ self . holding , self . holding ]
    self . protoSeqX , self . protoSeqY = [ 0 ] , [ self . holding ]
    return
# load our protocol from the header
proto = self . header [ 'dictEpochInfoPerDAC' ] [ self . channel ]
# prepare our ( x , y ) pair arrays
self . protoX , self . protoY = [ ] , [ ]
# assume our zero time point is the " holding " value
self . protoX . append ( 0 )
self . protoY . append ( self . holding )
# TODO : what is this ? ? ?
# now add x , y points for each change in the protocol
for step in proto :
    dX = proto [ step ] [ 'lEpochInitDuration' ]
    Y = proto [ step ] [ 'fEpochInitLevel' ] + proto [ step ] [ 'fEpochLevelInc' ] * self . sweep
    # we have a new Y value , so add it to the last time point
    self . protoX . append ( self . protoX [ - 1 ] )
    self . protoY . append ( Y )
    # now add the same Y point after " dX " amount of time
    self . protoX . append ( self . protoX [ - 1 ] + dX )
    self . protoY . append ( Y )
    # TODO : detect ramps and warn what ' s up
# The last point is probably holding current
finalVal = self . holding
# regular holding
# although if it ' s set to " use last value " , maybe that should be the last one
if self . header [ 'listDACInfo' ] [ 0 ] [ 'nInterEpisodeLevel' ] :
    finalVal = self . protoY [ - 1 ]
# add the shift to the final value to the list
self . protoX . append ( self . protoX [ - 1 ] )
self . protoY . append ( finalVal )
# and again as the very last time point
self . protoX . append ( self . sweepSize )
self . protoY . append ( finalVal )
# update the sequence of protocols now ( eliminate duplicate entries )
for i in range ( 1 , len ( self . protoX ) - 1 ) : # correct for weird ABF offset issue .
    self . protoX [ i ] = self . protoX [ i ] + self . offsetX
self . protoSeqY = [ self . protoY [ 0 ] ]
self . protoSeqX = [ self . protoX [ 0 ] ]
for i in range ( 1 , len ( self . protoY ) ) :
    if not self . protoY [ i ] == self . protoY [ i - 1 ] :
        self . protoSeqY . append ( self . protoY [ i ] )
        self . protoSeqX . append ( self . protoX [ i ] )
if self . protoY [ 0 ] != self . protoY [ 1 ] :
    self . protoY . insert ( 1 , self . protoY [ 0 ] )
    self . protoX . insert ( 1 , self . protoX [ 1 ] )
    self . protoY . insert ( 1 , self . protoY [ 0 ] )
    self . protoX . insert ( 1 , self . protoX [ 0 ] + self . offsetX / 2 )
self . protoSeqY . append ( finalVal )
self . protoSeqX . append ( self . sweepSize )
# convert lists to numpy arrays and do any final conversions
self . protoX = np . array ( self . protoX ) / self . pointsPerSec
self . protoY = np . array ( self . protoY )


signature:
def get_protocol ( self , sweep ) :
    """given a sweep , return the protocol as [ Xs , Ys ] .
 This is good for plotting / recreating the protocol trace .
 There may be duplicate numbers ."""

implementation:
self . setsweep ( sweep )
return list ( self . protoX ) , list ( self . protoY )


signature:
def get_protocol_sequence ( self , sweep ) :
    """given a sweep , return the protocol as condensed sequence .
 This is better for comparing similarities and determining steps .
 There should be no duplicate numbers ."""

implementation:
self . setsweep ( sweep )
return list ( self . protoSeqX ) , list ( self . protoSeqY )


signature:
def epochTimes ( self , nEpoch = 2 ) :
    """alternative to the existing abf protocol stuff
 return the start / stop time of an epoch .
 Epoch start at zero .
 A = 0 , B = 1 , C = 2 , D = 3 , . . ."""

implementation:
times = [ ]
durations = [ ]
for epoch in self . header [ 'dictEpochInfoPerDAC' ] [ self . channel ] . values ( ) :
    print ( epoch [ 'lEpochInitDuration' ] / self . pointsPerSec )
    times . append ( sum ( durations ) )
    durations . append ( epoch [ 'lEpochInitDuration' ] / self . pointsPerSec )
times . append ( sum ( durations ) )
times = np . array ( times ) + self . offsetX / self . pointsPerSec
# WHY ? ! ?
if nEpoch :
    return times [ nEpoch ] , times [ nEpoch + 1 ]
else :
    return times


signature:
def average ( self , t1 = 0 , t2 = None , setsweep = False ) :
    """return the average of part of the current sweep ."""

implementation:
if setsweep :
    self . setsweep ( setsweep )
if t2 is None or t2 > self . sweepLength :
    t2 = self . sweepLength
    self . log . debug ( "resetting t2 to [%f]" , t2 )
t1 = max ( t1 , 0 )
if t1 > t2 :
    self . log . error ( "t1 cannot be larger than t2" )
    return False
I1 , I2 = int ( t1 * self . pointsPerSec ) , int ( t2 * self . pointsPerSec )
if I1 == I2 :
    return np . nan
return np . average ( self . sweepY [ I1 : I2 ] )


signature:
def averageSweep ( self , sweepFirst = 0 , sweepLast = None ) :
    """Return a sweep which is the average of multiple sweeps .
 For now , standard deviation is lost ."""

implementation:
if sweepLast is None :
    sweepLast = self . sweeps - 1
nSweeps = sweepLast - sweepFirst + 1
runningSum = np . zeros ( len ( self . sweepY ) )
self . log . debug ( "averaging sweep %d to %d" , sweepFirst , sweepLast )
for sweep in np . arange ( nSweeps ) + sweepFirst :
    self . setsweep ( sweep )
    runningSum += self . sweepY . flatten ( )
average = runningSum / nSweeps
# TODO : standard deviation ?
return average


signature:
def kernel_gaussian ( self , sizeMS , sigmaMS = None , forwardOnly = False ) :
    """create kernel based on this ABF info ."""

implementation:
sigmaMS = sizeMS / 10 if sigmaMS is None else sigmaMS
size , sigma = sizeMS * self . pointsPerMs , sigmaMS * self . pointsPerMs
self . kernel = swhlab . common . kernel_gaussian ( size , sigma , forwardOnly )
return self . kernel


signature:
def sweepYfiltered ( self ) :
    """Get the filtered sweepY of the current sweep .
 Only works if self . kernel has been generated ."""

implementation:
assert self . kernel is not None
return swhlab . common . convolve ( self . sweepY , self . kernel )


signature:
def phasicNet ( self , biggestEvent = 50 , m1 = .5 , m2 = None ) :
    """Calculates the net difference between positive / negative phasic events
 Returns return the phasic difference value of the current sweep .
 Arguments :
 biggestEvent ( int ) : the size of the largest event anticipated
 m1 ( int , optional ) : the time ( sec ) to start analyzing
 m2 ( int , optional ) : the time ( sec ) to end analyzing
 Example :
 abf = swhlab . ABF ( abfFile )
 abf . kernel = abf . kernel _ gaussian ( sizeMS = 500 ) # kernel for smart baseline
 diff = [ ]
 for sweep in abf . setsweeps ( ) :
 print ( " Sweep " , sweep )
 diff . append ( analyzeSweep ( abf , plot = True , label = " sweep % d " % sweep ) )
 print ( diff )"""

implementation:
# determine marks ( between which we will analyze )
m1 = 0 if m1 is None else self . pointsPerSec * m1
m2 = - 1 if m2 is None else self . pointsPerSec * m2
# acquire the baseline - subtracted sweep
Y = self . sweepYsmartbase ( ) [ int ( m1 ) : int ( m2 ) ]
# create the histogram
nBins = 1000
hist , bins = np . histogram ( Y , bins = nBins , range = [ - biggestEvent , biggestEvent ] , density = True )
histSmooth = swhlab . common . lowpass ( hist )
# normalize height to 1
# TODO : should it be normalized first or not ?
# YES if reporting the ratio of the up / down area , NO if reporting the up - down difference
# hist , histSmooth = hist / max ( histSmooth ) , histSmooth / max ( histSmooth )
# center the peak at 0 pA
peakI = np . where ( histSmooth == max ( histSmooth ) ) [ 0 ] [ 0 ]
hist = np . roll ( hist , int ( nBins / 2 - peakI ) )
histSmooth = np . roll ( histSmooth , int ( nBins / 2 - peakI ) )
# calculate our mirrored difference
downward , upward = np . split ( histSmooth , 2 )
downward = downward [ : : - 1 ]
diff = np . sum ( upward - downward )
# convert our " pA / time " to " pA / sec "
diff = diff / ( len ( Y ) / self . pointsPerSec )
return diff


signature:
def output_touch ( self ) :
    """ensure the . / swhlab / folder exists ."""

implementation:
if not os . path . exists ( self . outFolder ) :
    self . log . debug ( "creating %s" , self . outFolder )
    os . mkdir ( self . outFolder )


signature:
def dictFlat ( l ) :
    """Given a list of list of dicts , return just the dicts ."""

implementation:
if type ( l ) is dict :
    return [ l ]
if "numpy" in str ( type ( l ) ) :
    return l
dicts = [ ]
for item in l :
    if type ( item ) == dict :
        dicts . append ( item )
    elif type ( item ) == list :
        for item2 in item :
            dicts . append ( item2 )
return dicts


signature:
def listCount ( l ) :
    """returns len ( ) of each item in a list , as a list ."""

implementation:
for i in range ( len ( l ) ) :
    l [ i ] = len ( l [ i ] )
return l


signature:
def dictVals ( l , key ) :
    """Return all ' key ' from a list of dicts . ( or list of list of dicts )"""

implementation:
dicts = dictFlat ( l )
vals = np . empty ( len ( dicts ) ) * np . nan
for i in range ( len ( dicts ) ) :
    if key in dicts [ i ] :
        vals [ i ] = dicts [ i ] [ key ]
return vals


signature:
def dictAvg ( listOfDicts , key , stdErr = False ) :
    """Given a list ( l ) of dicts ( d ) , return AV and SD ."""

implementation:
vals = dictVals ( listOfDicts , key )
if len ( vals ) and np . any ( vals ) :
    av = np . nanmean ( vals )
    er = np . nanstd ( vals )
    if stdErr :
        er = er / np . sqrt ( np . count_nonzero ( ~ np . isnan ( er ) ) )
else :
    av , er = np . nan , np . nan
return av , er


signature:
def dummyListOfDicts ( size = 100 ) :
    """returns a list ( of the given size ) of dicts with fake data .
 some dictionary keys are missing for some of the items ."""

implementation:
titles = "ahp,halfwidth,peak,expT,expI,sweep" . split ( "," )
ld = [ ]
# list of dicts
for i in range ( size ) :
    d = { }
    for t in titles :
        if int ( np . random . random ( 1 ) * 100 ) > 5 : # 5 % of values are missing
            d [ t ] = float ( np . random . random ( 1 ) * 100 )
            # random number 0-100
        if t == "sweep" and "sweep" in d . keys ( ) :
            d [ t ] = int ( d [ t ] )
    ld . append ( d )
return ld


signature:
def matrixValues ( matrix , key ) :
    """given a key , return a list of values from the matrix with that key ."""

implementation:
assert key in matrix . dtype . names
col = matrix . dtype . names . index ( key )
values = np . empty ( len ( matrix ) ) * np . nan
for i in range ( len ( matrix ) ) :
    values [ i ] = matrix [ i ] [ col ]
return values


signature:
def matrixToDicts ( data ) :
    """given a recarray , return it as a list of dicts ."""

implementation:
# 1D array
if "float" in str ( type ( data [ 0 ] ) ) :
    d = { }
    for x in range ( len ( data ) ) :
        d [ data . dtype . names [ x ] ] = data [ x ]
    return d
# 2D array
l = [ ]
for y in range ( len ( data ) ) :
    d = { }
    for x in range ( len ( data [ y ] ) ) :
        d [ data . dtype . names [ x ] ] = data [ y ] [ x ]
    l . append ( d )
return l


signature:
def matrixfromDicts ( dicts ) :
    """Give a list of dicts ( or list of list of dicts ) return a structured array .
 Headings will be sorted in alphabetical order ."""

implementation:
if 'numpy' in str ( type ( dicts ) ) :
    return dicts
    # already an array ?
names = set ( [ ] )
dicts = dictFlat ( dicts )
for item in dicts :
    names = names . union ( list ( item . keys ( ) ) )
names = sorted ( list ( names ) )
data = np . empty ( ( len ( dicts ) , len ( names ) ) , dtype = float ) * np . nan
for y in range ( len ( dicts ) ) :
    for key in dicts [ y ] . keys ( ) :
        for x in range ( len ( names ) ) :
            if names [ x ] in dicts [ y ] :
                data [ y , x ] = dicts [ y ] [ names [ x ] ]
if len ( dicts ) :
    data = np . core . records . fromarrays ( data . transpose ( ) , names = names )
return data


signature:
def htmlListToTR ( l , trClass = None , tdClass = None , td1Class = None ) :
    """turns a list into a < tr > < td > something < / td > < / tr >
 call this when generating HTML tables dynamically ."""

implementation:
html = "
"
for item in l :
    if 'array' in str ( type ( item ) ) :
        item = item [ 0 ]
        # TODO : why is this needed
    html += "%s	" % item
html += "
"
if trClass :
    html = html . replace ( "
" , '
' % trClass )
if td1Class :
    html = html . replace ( "" , '' % td1Class , 1 )
if tdClass :
    html = html . replace ( "" , '' % tdClass )
return html


signature:
def html_temp_launch ( html ) :
    """given text , make it a temporary HTML file and launch it ."""

implementation:
fname = tempfile . gettempdir ( ) + "/swhlab/temp.html"
with open ( fname , 'w' ) as f :
    f . write ( html )
webbrowser . open ( fname )


signature:
def checkOut ( thing , html = True ) :
    """show everything we can about an object ' s projects and methods ."""

implementation:
msg = ""
for name in sorted ( dir ( thing ) ) :
    if not "__" in name :
        msg += "%s\n" % name
        try :
            msg += " ^-VALUE: %s\n" % getattr ( thing , name ) ( )
        except :
            pass
if html :
    html = '' + msg + ''
    html = html . replace ( " " , " " ) . replace ( "\n" , "
" )
    fname = tempfile . gettempdir ( ) + "/swhlab/checkout.html"
    with open ( fname , 'w' ) as f :
        f . write ( html )
    webbrowser . open ( fname )
print ( msg . replace ( '' , '' ) . replace ( '' , '' ) )


signature:
def matrixToWks ( data , names = None , units = None , bookName = None , sheetName = " " , xCol = None ) :
    """Put 2d numpy data into an Origin worksheet .
 If bookname and sheetname are given try to load data into that book / sheet .
 If the book / sheet doesn ' t exist , create it ."""

implementation:
if type ( data ) is list :
    data = matrixfromDicts ( data )
if not names :
    names = [ "" ] * len ( data [ 0 ] )
    if data . dtype . names :
        names = list ( data . dtype . names )
if not units :
    units = [ "" ] * len ( data [ 0 ] )
    for i in range ( len ( units ) ) :
        if names [ i ] in UNITS . keys ( ) :
            units [ i ] = UNITS [ names [ i ] ]
if 'recarray' in str ( type ( data ) ) : # make it a regular array
    data = data . view ( float ) . reshape ( data . shape + ( - 1 , ) )
if xCol and xCol in names :
    xCol = names . index ( xCol )
    names . insert ( 0 , names [ xCol ] )
    units . insert ( 0 , units [ xCol ] )
    data = np . insert ( data , 0 , data [ : , xCol ] , 1 )
if not bookName :
    bookName = "tempBook"
if not sheetName :
    sheetName = "temp-" + str ( time . clock ( ) ) [ - 5 : ]
try :
    import PyOrigin
    PyOrigin . LT_execute ( "" )
    # try to crash a non - orign environment
except :
    print ( " -- PyOrigin not running, making HTML output." )
    matrixToHTML ( data , names , units , bookName , sheetName , xCol )
    return
nrows , ncols = len ( data ) , len ( data [ 0 ] )
if 'recarray' in str ( type ( data ) ) : # convert structured array to regular array
    data = np . array ( data . view ( ) , dtype = float ) . reshape ( ( nrows , ncols ) )
data = np . transpose ( data )
# transpose it
PyOrigin . LT_execute ( "activateSheet(%s, %s)" % ( bookName , sheetName ) )
wks = PyOrigin . ActiveLayer ( )
while wks . GetColCount ( ) < ncols :
    wks . InsertCol ( wks . GetColCount ( ) , '' )
for i in range ( ncols ) :
    col = wks . Columns ( i )
    col . SetLongName ( names [ i ] )
    col . SetUnits ( units [ i ] )
wks . SetData ( data , 0 , 0 )
PyOrigin . LT_execute ( "FixNanBug" )
PyOrigin . LT_execute ( "ABFPathToMetaData" )


signature:
def matrixToHTML ( data , names = None , units = None , bookName = None , sheetName = None , xCol = None ) :
    """Put 2d numpy data into a temporary HTML file ."""

implementation:
if not names :
    names = [ "" ] * len ( data [ 0 ] )
    if data . dtype . names :
        names = list ( data . dtype . names )
if not units :
    units = [ "" ] * len ( data [ 0 ] )
    for i in range ( len ( units ) ) :
        if names [ i ] in UNITS . keys ( ) :
            units [ i ] = UNITS [ names [ i ] ]
if 'recarray' in str ( type ( data ) ) : # make it a regular array
    data = data . view ( float ) . reshape ( data . shape + ( - 1 , ) )
if xCol and xCol in names :
    xCol = names . index ( xCol )
    names . insert ( 0 , names [ xCol ] )
    units . insert ( 0 , units [ xCol ] )
    data = np . insert ( data , 0 , data [ : , xCol ] , 1 )
htmlFname = tempfile . gettempdir ( ) + "/swhlab/WKS-%s.%s.html" % ( bookName , sheetName )
html = """

    """
html += "
FauxRigin
"
if bookName or sheetName :
    html += '%s / %s

' % ( bookName , sheetName )
html += "
"
# cols = list ( range ( len ( names ) ) )
colNames = [ '' ]
for i in range ( len ( units ) ) :
    label = "%s (%d)" % ( chr ( i + ord ( 'A' ) ) , i )
    colNames . append ( label )
html += htmlListToTR ( colNames , 'labelCol' , 'labelCol' )
html += htmlListToTR ( [ 'Long Name' ] + list ( names ) , 'name' , td1Class = 'labelRow' )
html += htmlListToTR ( [ 'Units' ] + list ( units ) , 'units' , td1Class = 'labelRow' )
cutOff = False
for y in range ( len ( data ) ) :
    html += htmlListToTR ( [ y + 1 ] + list ( data [ y ] ) , trClass = 'data%d' % ( y % 2 ) , td1Class = 'labelRow' )
    if y >= 200 :
        cutOff = True
        break
html += "
"
html = html . replace ( ">nan<" , ">--<" )
html = html . replace ( ">None<" , "><" )
if cutOff :
    html += "
... showing only %d of %d rows ...
" % ( y , len ( data ) )
html += ""
with open ( htmlFname , 'w' ) as f :
    f . write ( html )
webbrowser . open ( htmlFname )
return


signature:
def XMLtoPython ( xmlStr = r"C:\Apps\pythonModules\GSTemp.xml" ) :
    """given a string or a path to an XML file , return an XML object ."""

implementation:
# TODO : this absolute file path crazy stuff needs to stop !
if os . path . exists ( xmlStr ) :
    with open ( xmlStr ) as f :
        xmlStr = f . read ( )
print ( xmlStr )
print ( "DONE" )
return


signature:
def algo_exp ( x , m , t , b ) :
    """mono - exponential curve ."""

implementation: return m * np . exp ( - t * x ) + b


signature:
def fit_exp ( y , graphToo = False ) :
    """Exponential fit . Returns [ multiplier , t , offset , time constant ]"""

implementation:
x = np . arange ( len ( y ) )
try :
    params , cv = scipy . optimize . curve_fit ( algo_exp , x , y , p0 = ( 1 , 1e-6 , 1 ) )
except :
    print ( " !! curve fit failed (%.02f points)" % len ( x ) )
    return np . nan , np . nan , np . nan , np . nan
    # can ' t fit so little data !
m , t , b = params
tau = 1 / t
if graphToo :
    pylab . figure ( figsize = ( 6 , 4 ) )
    pylab . grid ( )
    pylab . title ( "FIT ASSESSMENT" )
    pylab . plot ( x , y , 'o' , mfc = 'none' , ms = 10 )
    pylab . plot ( x , algo_exp ( x , m , t , b ) , 'b-' , lw = 2 )
    pylab . show ( )
return m , t , b , tau


signature:
def numpyAlignXY ( data ) :
    """given a numpy array ( XYXYXY columns ) , return it aligned .
 data returned will be XYYY . NANs may be returned ."""

implementation:
print ( data )
Xs = data . flatten ( ) [ : : 2 ]
# get all X values
Xs = Xs [ ~ np . isnan ( Xs ) ]
# remove nans
Xs = sorted ( list ( set ( Xs ) ) )
# eliminate duplicates then sort it
aligned = np . empty ( ( len ( Xs ) , int ( len ( data [ 0 ] ) / 2 + 1 ) ) ) * np . nan
aligned [ : , 0 ] = Xs
for col in range ( 0 , len ( data [ 0 ] ) , 2 ) :
    for row in range ( len ( data ) ) :
        X = data [ row , col ]
        Y = data [ row , col + 1 ]
        if np . isnan ( X ) or np . isnan ( Y ) :
            continue
        aligned [ Xs . index ( X ) , int ( col / 2 + 1 ) ] = Y
return aligned


signature:
def filter_gaussian ( Ys , sigma , plotToo = False ) :
    """simple gaussian convolution . Returns same # of points as gotten ."""

implementation:
timeA = time . time ( )
window = scipy . signal . gaussian ( len ( Ys ) , sigma )
window /= sum ( window )
Ys2 = np . convolve ( Ys , window , 'same' )
print ( "LEN:" , len ( Ys2 ) , len ( Ys ) )
timeB = time . time ( )
print ( "convolution took %.03f ms" % ( ( timeB - timeA ) * 1000 ) )
if len ( Ys2 ) != len ( Ys ) :
    print ( "?!?!?!? convolution point size mismatch" )
if plotToo :
    pylab . plot ( Ys , label = 'original' , alpha = .2 )
    pylab . plot ( Ys2 , 'b-' , label = 'smooth' )
    pylab . legend ( )
    pylab . show ( )
return Ys2


signature:
def where_cross ( data , threshold ) :
    """return a list of Is where the data first crosses above threshold ."""

implementation:
Is = np . where ( data > threshold ) [ 0 ]
Is = np . concatenate ( ( [ 0 ] , Is ) )
Ds = Is [ : - 1 ] - Is [ 1 : ] + 1
return Is [ np . where ( Ds ) [ 0 ] + 1 ]


signature:
def show ( closeToo = False ) :
    """alternative to pylab . show ( ) that updates IPython window ."""

implementation:
IPython . display . display ( pylab . gcf ( ) )
if closeToo :
    pylab . close ( 'all' )


signature:
def originFormat_listOfDicts ( l ) :
    """Return [ { } , { } , { } ] as a 2d matrix ."""

implementation:
titles = [ ]
for d in l :
    for k in d . keys ( ) :
        if not k in titles :
            titles . append ( k )
titles . sort ( )
data = np . empty ( ( len ( l ) , len ( titles ) ) ) * np . nan
for y in range ( len ( l ) ) :
    for x in range ( len ( titles ) ) :
        if titles [ x ] in l [ y ] . keys ( ) :
            data [ y ] [ x ] = l [ y ] [ titles [ x ] ]
return titles , data


signature:
def originFormat ( thing ) :
    """Try to format anything as a 2D matrix with column names ."""

implementation:
if type ( thing ) is list and type ( thing [ 0 ] ) is dict :
    return originFormat_listOfDicts ( thing )
if type ( thing ) is list and type ( thing [ 0 ] ) is list :
    return originFormat_listOfDicts ( dictFlat ( thing ) )
else :
    print ( " !! I don't know how to format this object!" )
    print ( thing )


signature:
def pickle_save ( thing , fname ) :
    """save something to a pickle file"""

implementation:
pickle . dump ( thing , open ( fname , "wb" ) , pickle . HIGHEST_PROTOCOL )
return thing


signature:
def msgDict ( d , matching = None , sep1 = "=" , sep2 = "\n" , sort = True , cantEndWith = None ) :
    """convert a dictionary to a pretty formatted string ."""

implementation:
msg = ""
if "record" in str ( type ( d ) ) :
    keys = d . dtype . names
else :
    keys = d . keys ( )
if sort :
    keys = sorted ( keys )
for key in keys :
    if key [ 0 ] == "_" :
        continue
    if matching :
        if not key in matching :
            continue
    if cantEndWith and key [ - len ( cantEndWith ) ] == cantEndWith :
        continue
    if 'float' in str ( type ( d [ key ] ) ) :
        s = "%.02f" % d [ key ]
    else :
        s = str ( d [ key ] )
    if "object" in s :
        s = '<object>'
    msg += key + sep1 + s + sep2
return msg . strip ( )


signature:
def groupsFromKey ( keyFile = './key.txt' ) :
    """given a groups file , return a dict of groups .
 Example :
 # # # GROUP : TR
 16602083
 16608059
 # # # GROUP : TU
 16504000
 16507011"""

implementation:
groups = { }
thisGroup = "?"
with open ( keyFile ) as f :
    raw = f . read ( ) . split ( "\n" )
for line in raw :
    line = line . strip ( )
    if len ( line ) < 3 :
        continue
    if "### GROUP" in line :
        thisGroup = line . split ( ": " ) [ 1 ]
        groups [ thisGroup ] = [ ]
    else :
        groups [ thisGroup ] = groups [ thisGroup ] + [ line ]
return groups


signature:
def findRelevantData ( fileList , abfs ) :
    """return an abf of the * FIRST * of every type of thing ."""

implementation:
relevant = [ ]
things = { }
for abf in abfs :
    for fname in fileList :
        if abf in fname and not fname in relevant :
            relevant . append ( fname )
for item in sorted ( relevant ) :
    thing = os . path . basename ( item )
    if ".png" in thing :
        continue
    if not "_" in thing :
        continue
    thing = thing . split ( "_" ) [ - 1 ] . split ( "." ) [ 0 ]
    if not thing in things . keys ( ) : # prevent overwriting
        things [ thing ] = item
return things


signature:
def determineProtocol ( fname ) :
    """determine the comment cooked in the protocol ."""

implementation:
f = open ( fname , 'rb' )
raw = f . read ( 5000 )
# it should be in the first 5k of the file
f . close ( )
protoComment = "unknown"
if b"SWHLab4[" in raw :
    protoComment = raw . split ( b"SWHLab4[" ) [ 1 ] . split ( b"]" , 1 ) [ 0 ]
elif b"SWH[" in raw :
    protoComment = raw . split ( b"SWH[" ) [ 1 ] . split ( b"]" , 1 ) [ 0 ]
else :
    protoComment = "?"
if not type ( protoComment ) is str :
    protoComment = protoComment . decode ( "utf-8" )
return protoComment


signature:
def forwardSlash ( listOfFiles ) :
    """convert silly C : \\ names \\ like \\ this . txt to c : / names / like / this . txt"""

implementation:
for i , fname in enumerate ( listOfFiles ) :
    listOfFiles [ i ] = fname . replace ( "\\" , "/" )
return listOfFiles


signature:
def scanABFfolder ( abfFolder ) :
    """scan an ABF directory and subdirectory . Try to do this just once .
 Returns ABF files , SWHLab files , and groups ."""

implementation:
assert os . path . isdir ( abfFolder )
filesABF = forwardSlash ( sorted ( glob . glob ( abfFolder + "/*.*" ) ) )
filesSWH = [ ]
if os . path . exists ( abfFolder + "/swhlab4/" ) :
    filesSWH = forwardSlash ( sorted ( glob . glob ( abfFolder + "/swhlab4/*.*" ) ) )
groups = getABFgroups ( filesABF )
return filesABF , filesSWH , groups


signature:
def getParent ( abfFname ) :
    """given an ABF file name , return the ABF of its parent ."""

implementation:
child = os . path . abspath ( abfFname )
files = sorted ( glob . glob ( os . path . dirname ( child ) + "/*.*" ) )
parentID = abfFname
# its own parent
for fname in files :
    if fname . endswith ( ".abf" ) and fname . replace ( ".abf" , ".TIF" ) in files :
        parentID = os . path . basename ( fname ) . replace ( ".abf" , "" )
    if os . path . basename ( child ) in fname :
        break
return parentID


signature:
def getParent2 ( abfFname , groups ) :
    """given an ABF and the groups dict , return the ID of its parent ."""

implementation:
if ".abf" in abfFname :
    abfFname = os . path . basename ( abfFname ) . replace ( ".abf" , "" )
for parentID in groups . keys ( ) :
    if abfFname in groups [ parentID ] :
        return parentID
return abfFname


signature:
def getNotesForABF ( abfFile ) :
    """given an ABF , find the parent , return that line of experiments . txt"""

implementation:
parent = getParent ( abfFile )
parent = os . path . basename ( parent ) . replace ( ".abf" , "" )
expFile = os . path . dirname ( abfFile ) + "/experiment.txt"
if not os . path . exists ( expFile ) :
    return "no experiment file"
with open ( expFile ) as f :
    raw = f . readlines ( )
for line in raw :
    if line [ 0 ] == '~' :
        line = line [ 1 : ] . strip ( )
        if line . startswith ( parent ) :
            while "\t\t" in line :
                line = line . replace ( "\t\t" , "\t" )
            line = line . replace ( "\t" , "\n" )
            return line
return "experiment.txt found, but didn't contain %s" % parent


signature:
def getABFgroups ( files ) :
    """given a list of ALL files ( not just ABFs ) , return a dict [ ID ] = [ ID , ID , ID ] .
 Parents are determined if a . abf matches a . TIF .
 This is made to assign children files to parent ABF IDs ."""

implementation:
children = [ ]
groups = { }
for fname in sorted ( files ) :
    if fname . endswith ( ".abf" ) :
        if fname . replace ( ".abf" , ".TIF" ) in files : # TODO : cap sensitive
            if len ( children ) :
                groups [ children [ 0 ] ] = children
            children = [ os . path . basename ( fname ) [ : - 4 ] ]
        else :
            children . append ( os . path . basename ( fname ) [ : - 4 ] )
groups [ children [ 0 ] ] = children
# print ( " - - found % d groups of % d ABFs " % ( len ( groups ) , len ( files ) ) )
return groups


signature:
def getIDfileDict ( files ) :
    """given a list of files , return a dict [ ID ] = [ files ] .
 This is made to assign children files to parent ABF IDs ."""

implementation:
d = { }
orphans = [ ]
for fname in files :
    if fname . endswith ( ".abf" ) :
        d [ os . path . basename ( fname ) [ : - 4 ] ] = [ ]
for fname in files :
    if fname . endswith ( ".html" ) or fname . endswith ( ".txt" ) :
        continue
        # don ' t try to assign to an ABF
    if len ( os . path . basename ( fname ) . split ( "." ) [ 0 ] ) >= 8 :
        ID = os . path . basename ( fname ) [ : 8 ]
        # ABF ID ( first several chars )
    else :
        ID = os . path . basename ( fname ) . split ( "." ) [ 0 ]
        # short filename , just not extension
    if ID in d . keys ( ) :
        d [ ID ] = d [ ID ] + [ fname ]
    else :
        orphans . append ( os . path . basename ( fname ) )
        # print ( " ? ? orphan file " , ID , os . path . basename ( fname ) )
if orphans :
    print ( " ?? found %d orphan files" % len ( orphans ) )
return d


signature:
def getIDsFromFiles ( files ) :
    """given a path or list of files , return ABF IDs ."""

implementation:
if type ( files ) is str :
    files = glob . glob ( files + "/*.*" )
IDs = [ ]
for fname in files :
    if fname [ - 4 : ] . lower ( ) == '.abf' :
        ext = fname . split ( '.' ) [ - 1 ]
        IDs . append ( os . path . basename ( fname ) . replace ( '.' + ext , '' ) )
return sorted ( IDs )


signature:
def inspectABF ( abf = exampleABF , saveToo = False , justPlot = False ) :
    """May be given an ABF object or filename ."""

implementation:
pylab . close ( 'all' )
print ( " ~~ inspectABF()" )
if type ( abf ) is str :
    abf = swhlab . ABF ( abf )
swhlab . plot . new ( abf , forceNewFigure = True )
if abf . sweepInterval * abf . sweeps < 60 * 5 : # shorter than 5 minutes
    pylab . subplot ( 211 )
    pylab . title ( "%s [%s]" % ( abf . ID , abf . protoComment ) )
    swhlab . plot . sweep ( abf , 'all' )
    pylab . subplot ( 212 )
    swhlab . plot . sweep ( abf , 'all' , continuous = True )
    swhlab . plot . comments ( abf )
else :
    print ( " -- plotting as long recording" )
    swhlab . plot . sweep ( abf , 'all' , continuous = True , minutes = True )
    swhlab . plot . comments ( abf , minutes = True )
    pylab . title ( "%s [%s]" % ( abf . ID , abf . protoComment ) )
swhlab . plot . annotate ( abf )
if justPlot :
    return
if saveToo :
    path = os . path . split ( abf . fname ) [ 0 ]
    basename = os . path . basename ( abf . fname )
    pylab . savefig ( os . path . join ( path , "_" + basename . replace ( ".abf" , ".png" ) ) )
pylab . show ( )
return


signature:
def ftp_login ( folder = None ) :
    """return an " FTP " object after logging in ."""

implementation:
pwDir = os . path . realpath ( __file__ )
for i in range ( 3 ) :
    pwDir = os . path . dirname ( pwDir )
pwFile = os . path . join ( pwDir , "passwd.txt" )
print ( " -- looking for login information in:\n   [%s]" % pwFile )
try :
    with open ( pwFile ) as f :
        lines = f . readlines ( )
    username = lines [ 0 ] . strip ( )
    password = lines [ 1 ] . strip ( )
    print ( " -- found a valid username/password" )
except :
    print ( " -- password lookup FAILED." )
    username = TK_askPassword ( "FTP LOGIN" , "enter FTP username" )
    password = TK_askPassword ( "FTP LOGIN" , "enter password for %s" % username )
    if not username or not password :
        print ( " !! failed getting login info. aborting FTP effort." )
        return
print ( "      username:" , username )
print ( "      password:" , "*" * ( len ( password ) ) )
print ( " -- logging in to FTP ..." )
try :
    ftp = ftplib . FTP ( "swharden.com" )
    ftp . login ( username , password )
    if folder :
        ftp . cwd ( folder )
    return ftp
except :
    print ( " !! login failure !!" )
    return False


signature:
def ftp_folder_match ( ftp , localFolder , deleteStuff = True ) :
    """upload everything from localFolder into the current FTP folder ."""

implementation:
for fname in glob . glob ( localFolder + "/*.*" ) :
    ftp_upload ( ftp , fname )
return


signature:
def version_upload ( fname , username = "nibjb" ) :
    """Only scott should do this . Upload new version to site ."""

implementation:
print ( "popping up pasword window..." )
password = TK_askPassword ( "FTP LOGIN" , "enter password for %s" % username )
if not password :
    return
print ( "username:" , username )
print ( "password:" , "*" * ( len ( password ) ) )
print ( "connecting..." )
ftp = ftplib . FTP ( "swharden.com" )
ftp . login ( username , password )
print ( "successful login!" )
ftp . cwd ( "/software/swhlab/versions" )
# IMMEDIATELY GO HERE ! ! !
print ( "uploading" , os . path . basename ( fname ) )
ftp . storbinary ( "STOR " + os . path . basename ( fname ) , open ( fname , "rb" ) , 1024 )
# for binary files
print ( "disconnecting..." )
ftp . quit ( )


signature:
def TK_askPassword ( title = "input" , msg = "type here:" ) :
    """use the GUI to ask for a string ."""

implementation:
root = tkinter . Tk ( )
root . withdraw ( )
# hide tk window
root . attributes ( "-topmost" , True )
# always on top
root . lift ( )
# bring to top
value = tkinter . simpledialog . askstring ( title , msg )
root . destroy ( )
return value


signature:
def TK_message ( title , msg ) :
    """use the GUI to pop up a message ."""

implementation:
root = tkinter . Tk ( )
root . withdraw ( )
# hide tk window
root . attributes ( "-topmost" , True )
# always on top
root . lift ( )
# bring to top
tkinter . messagebox . showwarning ( title , msg )
root . destroy ( )


signature:
def TK_ask ( title , msg ) :
    """use the GUI to ask YES or NO ."""

implementation:
root = tkinter . Tk ( )
root . attributes ( "-topmost" , True )
# always on top
root . withdraw ( )
# hide tk window
result = tkinter . messagebox . askyesno ( title , msg )
root . destroy ( )
return result


signature:
def image_convert ( fname , saveAs = True , showToo = False ) :
    """Convert weird TIF files into web - friendly versions .
 Auto contrast is applied ( saturating lower and upper 0.1 % ) .
 make saveAs True to save as . TIF . png
 make saveAs False and it won ' t save at all
 make saveAs " someFile . jpg " to save it as a different path / format"""

implementation:
# load the image
# im = Image . open ( fname ) # PIL can ' t handle 12 - bit TIFs well
im = scipy . ndimage . imread ( fname )
# scipy does better with it
im = np . array ( im , dtype = float )
# now it ' s a numpy array
# do all image enhancement here
cutoffLow = np . percentile ( im , .01 )
cutoffHigh = np . percentile ( im , 99.99 )
im [ np . where ( im < cutoffLow ) ] = cutoffLow
im [ np . where ( im > cutoffHigh ) ] = cutoffHigh
# IMAGE FORMATTING
im -= np . min ( im )
# auto contrast
im /= np . max ( im )
# normalize
im *= 255
# stretch contrast ( 8 - bit )
im = Image . fromarray ( im )
# IMAGE DRAWING
msg = "Filename: %s\n" % os . path . basename ( fname )
timestamp = datetime . datetime . fromtimestamp ( os . path . getctime ( fname ) )
msg += "Created: %s\n" % timestamp . strftime ( '%Y-%m-%d %H:%M:%S' )
d = ImageDraw . Draw ( im )
fnt = ImageFont . truetype ( "arial.ttf" , 20 )
d . text ( ( 6 , 6 ) , msg , font = fnt , fill = 0 )
d . text ( ( 4 , 4 ) , msg , font = fnt , fill = 255 )
if showToo :
    im . show ( )
if saveAs is False :
    return
if saveAs is True :
    saveAs = fname + ".png"
im . convert ( 'RGB' ) . save ( saveAs )


signature:
def processArgs ( ) :
    """check out the arguments and figure out what to do ."""

implementation:
if len ( sys . argv ) < 2 :
    print ( "\n\nERROR:" )
    print ( "this script requires arguments!" )
    print ( 'try "python command.py info"' )
    return
if sys . argv [ 1 ] == 'info' :
    print ( "import paths:\n " , "\n  " . join ( sys . path ) )
    print ( )
    print ( "python version:" , sys . version )
    print ( "SWHLab path:" , __file__ )
    print ( "SWHLab version:" , swhlab . __version__ )
    return
if sys . argv [ 1 ] == 'glanceFolder' :
    abfFolder = swhlab . common . gui_getFolder ( )
    if not abfFolder or not os . path . isdir ( abfFolder ) :
        print ( "bad path" )
        return
    fnames = sorted ( glob . glob ( abfFolder + "/*.abf" ) )
    outFolder = tempfile . gettempdir ( ) + "/swhlab/"
    if os . path . exists ( outFolder ) :
        shutil . rmtree ( outFolder )
    os . mkdir ( outFolder )
    outFile = outFolder + "/index.html"
    out = ''
    out += '
%s
' % abfFolder
    for i , fname in enumerate ( fnames ) :
        print ( "\n\n### PROCESSING %d of %d" % ( i , len ( fnames ) ) )
        saveAs = os . path . join ( os . path . dirname ( outFolder ) , os . path . basename ( fname ) ) + ".png"
        out += '

%s
' % os . path . abspath ( fname )
        out += '
' % ( saveAs , saveAs )
        swhlab . analysis . glance . processAbf ( fname , saveAs )
    out += ''
    with open ( outFile , 'w' ) as f :
        f . write ( out )
    webbrowser . open_new_tab ( outFile )
    return
print ( "\n\nERROR:\nI'm not sure how to process these arguments!" )
print ( sys . argv )


signature:
def detect ( abf , sweep = None , threshold_upslope = 50 , dT = .1 , saveToo = True ) :
    """An AP will be detected by a upslope that exceeds 50V / s . Analyzed too .
 if type ( sweep ) is int , graph int ( sweep )
 if sweep = = None , process all sweeps sweep ."""

implementation:
if type ( sweep ) is int :
    sweeps = [ sweep ]
else :
    sweeps = list ( range ( abf . sweeps ) )
timeStart = time . clock ( )
abf . APs = [ None ] * abf . sweeps
abf . SAP = [ None ] * abf . sweeps
for sweep in sweeps :
    abf . setSweep ( sweep )
    Y = abf . dataY
    dI = int ( dT / 1000 * abf . rate )
    # dI is dT / rate
    dY = ( Y [ dI : ] - Y [ : - dI ] ) * ( abf . rate / 1000 / dI )
    # now in V / S
    Is = cm . where_cross ( dY , threshold_upslope )
    # found every putative AP ( I units )
    abf . APs [ sweep ] = [ ]
    for i in range ( len ( Is ) ) : # for each putative AP
        try :
            AP = analyzeAP ( Y , dY , Is [ i ] , abf . rate )
            # try to characterize it
            if AP :
                AP [ "sweep" ] = sweep
                AP [ "expI" ] = sweep * abf . sweepInterval * abf . rate * + AP [ "sweepI" ]
                AP [ "expT" ] = sweep * abf . sweepInterval + AP [ "sweepT" ]
                AP [ "freq" ] = np . nan
                # default
                if len ( abf . APs [ sweep ] ) :
                    AP [ "freq" ] = 1 / ( AP [ "expT" ] - abf . APs [ sweep ] [ - 1 ] [ "expT" ] )
                if AP [ "freq" ] is np . nan or AP [ "freq" ] < 500 : # at 500Hz , assume you have a duplicate AP
                    abf . APs [ sweep ] . append ( AP )
        except :
            print ( " -- AP %d of %d excluded from analysis..." % ( i + 1 , len ( Is ) ) )
            # print ( " ! ! ! AP CRASH ! ! ! " )
            # print ( traceback . format _ exc ( ) )
    analyzeAPgroup ( abf )
    # now that APs are known , get grouping stats
abf . APs = cm . matrixfromDicts ( abf . APs )
abf . SAP = cm . matrixfromDicts ( abf . SAP )
print ( " -- analyzed %d APs in %.02f ms" % ( len ( cm . dictFlat ( abf . APs ) ) , ( time . clock ( ) - timeStart ) * 1000 ) )
if saveToo :
    abf . saveThing ( abf . APs , "APs" )
    abf . saveThing ( abf . SAP , "SAP" )


signature:
def analyzeAPgroup ( abf = exampleABF , T1 = None , T2 = None , plotToo = False ) :
    """On the current ( setSweep ( ) ) sweep , calculate things like accomodation .
 Only call directly just for demonstrating how it works by making a graph .
 Or call this if you want really custom T1 and T2 ( multiple per sweep )
 This is called by default with default T1 and T2.
 Manually call it again for custom ."""

implementation:
if T1 is None or T2 is None :
    if len ( abf . protoSeqX ) > 2 :
        T1 = abf . protoSeqX [ 1 ] / abf . rate
        T2 = abf . protoSeqX [ 2 ] / abf . rate
    else :
        T1 = 0
        T2 = abf . sweepLength
s = { }
# sweep dictionary to contain our stas
s [ "sweep" ] = abf . currentSweep
s [ "commandI" ] = abf . protoSeqY [ 1 ]
APs = [ ]
for key in [ 'freqAvg' , 'freqBin' ] :
    s [ key ] = 0
for AP in abf . APs [ abf . currentSweep ] :
    if T1 < AP [ "sweepT" ] < T2 :
        APs . append ( AP )
s [ "nAPs" ] = len ( APs )
# number of APs in the bin period ( T1 - T2)
apTimes = cm . dictVals ( APs , 'sweepT' )
if len ( APs ) > 1 : # some measurements require multiple APs , like accomodation
    s [ "centerBinTime" ] = np . average ( apTimes ) - T1
    # average time of APs in the bin
    s [ "centerBinFrac" ] = s [ "centerBinTime" ] / ( T2 - T1 ) * 100
    # fractional average of APs in bin ( steady = . 5)
    s [ "centerTime" ] = np . average ( apTimes ) - APs [ 0 ] [ "sweepT" ]
    # time of average AP WRT first AP ( not bin )
    s [ "centerFrac" ] = s [ "centerTime" ] / ( APs [ - 1 ] [ "sweepT" ] - APs [ 0 ] [ "sweepT" ] ) * 100
    # WRT first / last AP
    s [ "msToFirst" ] = ( APs [ 0 ] [ "sweepT" ] - T1 ) * 1000
    # ms to first AP ( from T1)
    s [ "freqFirst1" ] = APs [ 1 ] [ 'freq' ]
    # inst frequency of first AP
    s [ "freqFirst5" ] = cm . dictAvg ( APs [ 1 : 6 ] , 'freq' ) [ 0 ]
    # inst frequency of first AP
    s [ "freqLast" ] = APs [ - 1 ] [ 'freq' ]
    # inst frequency of last AP
    s [ "freqAvg" ] = cm . dictAvg ( APs , 'freq' ) [ 0 ]
    # average inst frequency of all aps
    s [ "freqBin" ] = len ( APs ) / ( T2 - T1 )
    # frequency of APs in the bin ( T1 - T2)
    s [ "freqSteady25" ] = cm . dictAvg ( APs [ - int ( len ( APs ) * .25 ) : ] , 'freq' ) [ 0 ]
    # average freq of the last 25 % of APs
    s [ "accom1Avg" ] = s [ "freqFirst1" ] / s [ "freqAvg" ]
    # accomodation ( first II / average )
    s [ "accom1Steady25" ] = s [ "freqFirst1" ] / s [ "freqSteady25" ]
    # accomodation ( first II / steady state )
    s [ "accom5Avg" ] = s [ "freqFirst5" ] / s [ "freqAvg" ]
    # accomodation from average 5 first
    s [ "accom5Steady25" ] = s [ "freqFirst5" ] / s [ "freqSteady25" ]
    # accomodation from average 5 first
    s [ "freqCV" ] = cm . dictAvg ( APs , 'freq' ) [ 1 ] / cm . dictAvg ( APs , 'freq' ) [ 0 ]
    # coefficient of variation ( Hz )
    s [ "T1" ] = T1
    s [ "T2" ] = T2
abf . SAP [ abf . currentSweep ] = s


signature:
def check_AP_group ( abf = exampleABF , sweep = 0 ) :
    """after running detect ( ) and abf . SAP is populated , this checks it ."""

implementation:
abf . setSweep ( sweep )
swhlab . plot . new ( abf , title = "sweep %d (%d pA)" % ( abf . currentSweep , abf . protoSeqY [ 1 ] ) )
swhlab . plot . sweep ( abf )
SAP = cm . matrixToDicts ( abf . SAP [ sweep ] )
if "T1" in SAP . keys ( ) :
    T1 = SAP [ "T1" ]
    T2 = SAP [ "T2" ]
    pylab . axvspan ( T1 / abf . rate , T2 / abf . rate , color = 'r' , alpha = .1 )
else :
    T1 = 0
    T2 = abf . sweepLength
swhlab . plot . annotate ( abf )
pylab . tight_layout ( )
pylab . subplots_adjust ( right = 0.6 )
pylab . annotate ( cm . msgDict ( SAP ) , ( .71 , .95 ) , ha = 'left' , va = 'top' , weight = 'bold' , family = 'monospace' , xycoords = 'figure fraction' , size = 12 , color = 'g' )
pylab . axis ( [ T1 - .05 , T2 + .05 , None , None ] )


signature:
def analyzeAP ( Y , dY , I , rate , verbose = False ) :
    """given a sweep and a time point , return the AP array for that AP .
 APs will be centered in time by their maximum upslope ."""

implementation:
Ims = int ( rate / 1000 )
# Is per MS
IsToLook = 5 * Ims
# TODO : clarify this , ms until downslope is over
upslope = np . max ( dY [ I : I + IsToLook ] )
# maximum rise velocity
upslopeI = np . where ( dY [ I : I + IsToLook ] == upslope ) [ 0 ] [ 0 ] + I
I = upslopeI
# center sweep at the upslope
downslope = np . min ( dY [ I : I + IsToLook ] )
# maximum fall velocity
downslopeI = np . where ( dY [ I : I + IsToLook ] == downslope ) [ 0 ] [ 0 ] + I
peak = np . max ( Y [ I : I + IsToLook ] )
# find peak value ( mV )
peakI = np . where ( Y [ I : I + IsToLook ] == peak ) [ 0 ] [ 0 ] + I
# find peak I
thresholdI = I - np . where ( dY [ I : I + IsToLook : - - 1 ] < 10 ) [ 0 ]
# detect < 10V / S
if not len ( thresholdI ) :
    return False
thresholdI = thresholdI [ 0 ]
threshold = Y [ thresholdI ]
# mV where > 10mV / S
height = peak - threshold
# height ( mV ) from threshold to peak
halfwidthPoint = np . average ( ( threshold , peak ) )
halfwidth = np . where ( Y [ I - IsToLook : I + IsToLook ] > halfwidthPoint ) [ 0 ]
if not len ( halfwidth ) :
    return False
    # doesn ' t look like a real AP
halfwidthI1 = halfwidth [ 0 ] + I - IsToLook
halfwidthI2 = halfwidth [ - 1 ] + I - IsToLook
if Y [ halfwidthI1 - 1 ] > halfwidthPoint or Y [ halfwidthI2 + 1 ] > halfwidthPoint :
    return False
    # doesn ' t look like a real AP
halfwidth = len ( halfwidth ) / rate * 1000
# now in MS
riseTime = ( peakI - thresholdI ) * 1000 / rate
# time ( ms ) from threshold to peak
IsToLook = 100 * Ims
# TODO : max prediction until AHP reaches nadir
AHPchunk = np . diff ( Y [ downslopeI : downslopeI + IsToLook ] )
# first inflection
AHPI = np . where ( AHPchunk > 0 ) [ 0 ]
if len ( AHPI ) == 0 :
    AHPI = np . nan
else :
    AHPI = AHPI [ 0 ] + downslopeI
    AHPchunk = Y [ AHPI : AHPI + IsToLook ]
    if max ( AHPchunk ) > threshold : # if another AP is coming , cut it out
        AHPchunk = AHPchunk [ : np . where ( AHPchunk > threshold ) [ 0 ] [ 0 ] ]
    if len ( AHPchunk ) :
        AHP = np . nanmin ( AHPchunk )
        AHPI = np . where ( AHPchunk == AHP ) [ 0 ] [ 0 ] + AHPI
        AHPheight = threshold - AHP
        # AHP magnitude from threshold ( mV )
        IsToLook = 500 * Ims
        # TODO : max prediction until AHP reaches threshold
        AHPreturn = np . average ( ( AHP , threshold ) )
        # half of threshold
        AHPreturnI = np . where ( Y [ AHPI : AHPI + IsToLook ] > AHPreturn ) [ 0 ]
        if len ( AHPreturnI ) : # not having a clean decay won ' t cause AP to crash
            AHPreturnI = AHPreturnI [ 0 ] + AHPI
            AHPrisetime = ( AHPreturnI - AHPI ) * 2 / rate * 1000
            # predicted return time ( ms )
            AHPupslope = AHPheight / AHPrisetime
            # mV / ms = V / S
            AHPreturnFullI = ( AHPreturnI - AHPI ) * 2 + AHPI
        else : # make them nan so you can do averages later
            AHPreturnI , AHPrisetime , AHPupslope = np . nan , np . nan , np . nan
            downslope = np . nan
# fasttime ( 10V / S to 10V / S ) # TODO :
# dpp ( deriv peak to peak ) # TODO :
sweepI , sweepT = I , I / rate
# clean up variable names
del IsToLook , I , Y , dY , Ims , AHPchunk , verbose
# delete what we don ' t need
return locals ( )


signature:
def check_sweep ( abf , sweep = None , dT = .1 ) :
    """Plotting for an eyeball check of AP detection in the given sweep ."""

implementation:
if abf . APs is None :
    APs = [ ]
else :
    APs = cm . matrixToDicts ( abf . APs )
if sweep is None or len ( sweep ) == 0 : # find the first sweep with > 5APs in it
    for sweepNum in range ( abf . sweeps ) :
        foundInThisSweep = 0
        for AP in APs :
            if AP [ "sweep" ] == sweepNum :
                foundInThisSweep += 1
            if foundInThisSweep >= 5 :
                break
    sweep = sweepNum
abf . setSweep ( sweep )
Y = abf . dataY
dI = int ( dT / 1000 * abf . rate )
# dI is dT / rate
dY = ( Y [ dI : ] - Y [ : - dI ] ) * ( abf . rate / 1000 / dI )
# now in V / S
pylab . figure ( figsize = ( 12 , 6 ) )
ax = pylab . subplot ( 211 )
pylab . title ( "sweep %d" % abf . currentSweep )
pylab . ylabel ( "membrane potential (mV)" )
pylab . plot ( Y , '-' , alpha = .8 )
for AP in APs :
    if not AP [ "sweep" ] == sweep :
        continue
    pylab . axvline ( AP [ "sweepI" ] , alpha = .2 , color = 'r' )
    pylab . plot ( AP [ "peakI" ] , AP [ "peak" ] , '.' , alpha = .5 , ms = 20 , color = 'r' )
    pylab . plot ( AP [ "thresholdI" ] , AP [ "threshold" ] , '.' , alpha = .5 , ms = 20 , color = 'c' )
    pylab . plot ( [ AP [ "AHPI" ] , AP [ "AHPreturnI" ] ] , [ AP [ "AHP" ] , AP [ "AHPreturn" ] ] , '-' , alpha = .2 , ms = 20 , color = 'b' , lw = 7 )
    pylab . plot ( [ AP [ "halfwidthI1" ] , AP [ "halfwidthI2" ] ] , [ AP [ "halfwidthPoint" ] , AP [ "halfwidthPoint" ] ] , '-' , lw = 5 , alpha = .5 , color = 'g' )
pylab . subplot ( 212 , sharex = ax )
pylab . ylabel ( "velocity (V/S)" )
pylab . xlabel ( "data points (%.02f kHz)" % ( abf . rate / 1000 ) )
pylab . plot ( dY , '-' , alpha = .8 )
pylab . margins ( 0 , .1 )
for AP in APs :
    if not AP [ "sweep" ] == sweep :
        continue
    pylab . axvline ( AP [ "sweepI" ] , alpha = .2 , color = 'r' )
    pylab . plot ( AP [ "upslopeI" ] , AP [ "upslope" ] , '.' , alpha = .5 , ms = 20 , color = 'g' )
    pylab . plot ( AP [ "downslopeI" ] , AP [ "downslope" ] , '.' , alpha = .5 , ms = 20 , color = 'g' )
    pylab . axis ( [ APs [ 0 ] [ "sweepI" ] - 1000 , APs [ - 1 ] [ "sweepI" ] + 1000 , None , None ] )


signature:
def get_AP_timepoints ( abf ) :
    """return list of time points ( sec ) of all AP events in experiment ."""

implementation:
col = abf . APs . dtype . names . index ( "expT" )
timePoints = [ ]
for i in range ( len ( abf . APs ) ) :
    timePoints . append ( abf . APs [ i ] [ col ] )
return timePoints


signature:
def stats_first ( abf ) :
    """provide all stats on the first AP ."""

implementation:
msg = ""
for sweep in range ( abf . sweeps ) :
    for AP in abf . APs [ sweep ] :
        for key in sorted ( AP . keys ( ) ) :
            if key [ - 1 ] is "I" or key [ - 2 : ] in [ "I1" , "I2" ] :
                continue
            msg += "%s = %s\n" % ( key , AP [ key ] )
        return msg


signature:
def get_values ( abf , key = "freq" , continuous = False ) :
    """returns Xs , Ys ( the key ) , and sweep # s for every AP found ."""

implementation:
Xs , Ys , Ss = [ ] , [ ] , [ ]
for sweep in range ( abf . sweeps ) :
    for AP in cm . matrixToDicts ( abf . APs ) :
        if not AP [ "sweep" ] == sweep :
            continue
        Ys . append ( AP [ key ] )
        Ss . append ( AP [ "sweep" ] )
        if continuous :
            Xs . append ( AP [ "expT" ] )
        else :
            Xs . append ( AP [ "sweepT" ] )
return np . array ( Xs ) , np . array ( Ys ) , np . array ( Ss )


signature:
def getAvgBySweep ( abf , feature , T0 = None , T1 = None ) :
    """return average of a feature divided by sweep ."""

implementation:
if T1 is None :
    T1 = abf . sweepLength
if T0 is None :
    T0 = 0
data = [ np . empty ( ( 0 ) ) ] * abf . sweeps
for AP in cm . dictFlat ( cm . matrixToDicts ( abf . APs ) ) :
    if T0 < AP [ 'sweepT' ] < T1 :
        val = AP [ feature ]
        data [ int ( AP [ 'sweep' ] ) ] = np . concatenate ( ( data [ int ( AP [ 'sweep' ] ) ] , [ val ] ) )
for sweep in range ( abf . sweeps ) :
    if len ( data [ sweep ] ) > 1 and np . any ( data [ sweep ] ) :
        data [ sweep ] = np . nanmean ( data [ sweep ] )
    elif len ( data [ sweep ] ) == 1 :
        data [ sweep ] = data [ sweep ] [ 0 ]
    else :
        data [ sweep ] = np . nan
return data


signature:
def readLog ( fname = "workdays.csv" , onlyAfter = datetime . datetime ( year = 2017 , month = 1 , day = 1 ) ) :
    """return a list of [ stamp , project ] elements ."""

implementation:
with open ( fname ) as f :
    raw = f . read ( ) . split ( "\n" )
efforts = [ ]
# date , nickname
for line in raw [ 1 : ] :
    line = line . strip ( ) . split ( "," )
    date = datetime . datetime . strptime ( line [ 0 ] , "%Y-%m-%d" )
    if onlyAfter and date < onlyAfter :
        continue
    if len ( line ) < 3 :
        continue
    for project in line [ 2 : ] :
        project = project . strip ( )
        if len ( project ) :
            efforts . append ( [ date , project ] )
return efforts


signature:
def waitTillCopied ( fname ) :
    """sometimes a huge file takes several seconds to copy over .
 This will hang until the file is copied ( file size is stable ) ."""

implementation:
lastSize = 0
while True :
    thisSize = os . path . getsize ( fname )
    print ( "size:" , thisSize )
    if lastSize == thisSize :
        print ( "size: STABLE" )
        return
    else :
        lastSize = thisSize
    time . sleep ( .1 )


signature:
def lazygo ( watchFolder = '../abfs/' , reAnalyze = False , rebuildSite = False , keepGoing = True , matching = False ) :
    """continuously monitor a folder for new abfs and try to analyze them .
 This is intended to watch only one folder , but can run multiple copies ."""

implementation:
abfsKnown = [ ]
while True :
    print ( )
    pagesNeeded = [ ]
    for fname in glob . glob ( watchFolder + "/*.abf" ) :
        ID = os . path . basename ( fname ) . replace ( ".abf" , "" )
        if not fname in abfsKnown :
            if os . path . exists ( fname . replace ( ".abf" , ".rsv" ) ) : # TODO : or something like this
                continue
            if matching and not matching in fname :
                continue
            abfsKnown . append ( fname )
            if os . path . exists ( os . path . dirname ( fname ) + "/swhlab4/" + os . path . basename ( fname ) . replace ( ".abf" , "_info.pkl" ) ) and reAnalyze == False :
                print ( "already analyzed" , os . path . basename ( fname ) )
                if rebuildSite :
                    pagesNeeded . append ( ID )
            else :
                handleNewABF ( fname )
                pagesNeeded . append ( ID )
    if len ( pagesNeeded ) :
        print ( " -- rebuilding index page" )
        indexing . genIndex ( os . path . dirname ( fname ) , forceIDs = pagesNeeded )
    if not keepGoing :
        return
    for i in range ( 50 ) :
        print ( '.' , end = '' )
        time . sleep ( .2 )


signature:
def phasicTonic ( self , m1 = None , m2 = None , chunkMs = 50 , quietPercentile = 10 , histResolution = .5 , plotToo = False , rmsExpected = 5 ) :
    """chunkMs should be ~ 50 ms or greater .
 bin sizes must be equal to or multiples of the data resolution .
 transients smaller than the expected RMS will be silenced ."""

implementation:
# prepare sectioning values to be used later
m1 = 0 if m1 is None else m1 * self . pointsPerSec
m2 = len ( abf . sweepY ) if m2 is None else m2 * self . pointsPerSec
m1 , m2 = int ( m1 ) , int ( m2 )
# prepare histogram values to be used later
padding = 200
# pA or mV of maximum expected deviation
chunkPoints = int ( chunkMs * self . pointsPerMs )
histBins = int ( ( padding * 2 ) / histResolution )
# center the data at 0 using peak histogram , not the mean
Y = self . sweepY [ m1 : m2 ]
hist , bins = np . histogram ( Y , bins = 2 * padding )
Yoffset = bins [ np . where ( hist == max ( hist ) ) [ 0 ] [ 0 ] ]
Y = Y - Yoffset
# we don ' t have to , but PDF math is easier
# calculate all histogram
nChunks = int ( len ( Y ) / chunkPoints )
hist , bins = np . histogram ( Y , bins = histBins , range = ( - padding , padding ) )
hist = hist / len ( Y )
# count as a fraction of total
Xs = bins [ 1 : ]
# get baseline data from chunks with smallest variance
chunks = np . reshape ( Y [ : nChunks * chunkPoints ] , ( nChunks , chunkPoints ) )
variances = np . var ( chunks , axis = 1 )
percentiles = np . empty ( len ( variances ) )
for i , variance in enumerate ( variances ) :
    percentiles [ i ] = sorted ( variances ) . index ( variance ) / len ( variances ) * 100
blData = chunks [ np . where ( percentiles <= quietPercentile ) [ 0 ] ] . flatten ( )
# generate the standard curve and pull it to the histogram height
sigma = np . sqrt ( np . var ( blData ) )
center = np . average ( blData ) + histResolution / 2
blCurve = mlab . normpdf ( Xs , center , sigma )
blCurve = blCurve * max ( hist ) / max ( blCurve )
# determine the phasic current by subtracting - out the baseline
diff = hist - blCurve
# manually zero - out data which we expect to be within the RMS range
ignrCenter = len ( Xs ) / 2
ignrPad = rmsExpected / histResolution
ignr1 , ignt2 = int ( ignrCenter - ignrPad ) , int ( ignrCenter + ignrPad )
diff [ ignr1 : ignt2 ] = 0
return diff / len ( Y ) * abf . pointsPerSec