signature:
def variability_threshold ( featuresdir , outfile , magbins = DEFAULT_MAGBINS , maxobjects = None , timecols = None , magcols = None , errcols = None , lcformat = 'hat-sql' , lcformatdir = None , min_lcmad_stdev = 5.0 , min_stetj_stdev = 2.0 , min_iqr_stdev = 2.0 , min_inveta_stdev = 2.0 , verbose = True ) :
    '''This generates a list of objects with stetson J , IQR , and 1.0 / eta
 above some threshold value to select them as potential variable stars .
 Use this to pare down the objects to review and put through
 period - finding . This does the thresholding per magnitude bin ; this should be
 better than one single cut through the entire magnitude range . Set the
 magnitude bins using the magbins kwarg .
 FIXME : implement a voting classifier here . this will choose variables based
 on the thresholds in IQR , stetson , and inveta based on weighting carried
 over from the variability recovery sims .
 Parameters
 featuresdir : str
 This is the directory containing variability feature pickles created by
 : py : func : ` astrobase . lcproc . lcpfeatures . parallel _ varfeatures ` or similar .
 outfile : str
 This is the output pickle file that will contain all the threshold
 information .
 magbins : np . array of floats
 This sets the magnitude bins to use for calculating thresholds .
 maxobjects : int or None
 This is the number of objects to process . If None , all objects with
 feature pickles in ` featuresdir ` will be processed .
 timecols : list of str or None
 The timecol keys to use from the lcdict in calculating the thresholds .
 magcols : list of str or None
 The magcol keys to use from the lcdict in calculating the thresholds .
 errcols : list of str or None
 The errcol keys to use from the lcdict in calculating the thresholds .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 min _ lcmad _ stdev , min _ stetj _ stdev , min _ iqr _ stdev , min _ inveta _ stdev : float or np . array
 These are all the standard deviation multiplier for the distributions of
 light curve standard deviation , Stetson J variability index , the light
 curve interquartile range , and 1 / eta variability index
 respectively . These multipliers set the minimum values of these measures
 to use for selecting variable stars . If provided as floats , the same
 value will be used for all magbins . If provided as np . arrays of ` size =
 magbins . size - 1 ` , will be used to apply possibly different sigma cuts
 for each magbin .
 verbose : bool
 If True , will report progress and warn about any problems .
 Returns
 dict
 Contains all of the variability threshold information along with indices
 into the array of the object IDs chosen as variables .'''

implementation:
try :
    formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
    if formatinfo :
        ( dfileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
    else :
        LOGERROR ( "can't figure out the light curve format" )
        return None
except Exception as e :
    LOGEXCEPTION ( "can't figure out the light curve format" )
    return None
# override the default timecols , magcols , and errcols
# using the ones provided to the function
if timecols is None :
    timecols = dtimecols
if magcols is None :
    magcols = dmagcols
if errcols is None :
    errcols = derrcols
# list of input pickles generated by varfeatures functions above
pklist = glob . glob ( os . path . join ( featuresdir , 'varfeatures-*.pkl' ) )
if maxobjects :
    pklist = pklist [ : maxobjects ]
allobjects = { }
for magcol in magcols : # keep local copies of these so we can fix them independently in case of
# nans
    if ( isinstance ( min_stetj_stdev , list ) or isinstance ( min_stetj_stdev , np . ndarray ) ) :
        magcol_min_stetj_stdev = min_stetj_stdev [ : : ]
    else :
        magcol_min_stetj_stdev = min_stetj_stdev
    if ( isinstance ( min_iqr_stdev , list ) or isinstance ( min_iqr_stdev , np . ndarray ) ) :
        magcol_min_iqr_stdev = min_iqr_stdev [ : : ]
    else :
        magcol_min_iqr_stdev = min_iqr_stdev
    if ( isinstance ( min_inveta_stdev , list ) or isinstance ( min_inveta_stdev , np . ndarray ) ) :
        magcol_min_inveta_stdev = min_inveta_stdev [ : : ]
    else :
        magcol_min_inveta_stdev = min_inveta_stdev
    LOGINFO ( 'getting all object sdssr, LC MAD, stet J, IQR, eta...' )
    # we ' ll calculate the sigma per magnitude bin , so get the mags as well
    allobjects [ magcol ] = { 'objectid' : [ ] , 'sdssr' : [ ] , 'lcmad' : [ ] , 'stetsonj' : [ ] , 'iqr' : [ ] , 'eta' : [ ] }
    # fancy progress bar with tqdm if present
    if TQDM and verbose :
        listiterator = tqdm ( pklist )
    else :
        listiterator = pklist
    for pkl in listiterator :
        with open ( pkl , 'rb' ) as infd :
            thisfeatures = pickle . load ( infd )
        objectid = thisfeatures [ 'objectid' ]
        # the object magnitude
        if ( 'info' in thisfeatures and thisfeatures [ 'info' ] and 'sdssr' in thisfeatures [ 'info' ] ) :
            if ( thisfeatures [ 'info' ] [ 'sdssr' ] and thisfeatures [ 'info' ] [ 'sdssr' ] > 3.0 ) :
                sdssr = thisfeatures [ 'info' ] [ 'sdssr' ]
            elif ( magcol in thisfeatures and thisfeatures [ magcol ] and 'median' in thisfeatures [ magcol ] and thisfeatures [ magcol ] [ 'median' ] > 3.0 ) :
                sdssr = thisfeatures [ magcol ] [ 'median' ]
            elif ( thisfeatures [ 'info' ] [ 'jmag' ] and thisfeatures [ 'info' ] [ 'hmag' ] and thisfeatures [ 'info' ] [ 'kmag' ] ) :
                sdssr = jhk_to_sdssr ( thisfeatures [ 'info' ] [ 'jmag' ] , thisfeatures [ 'info' ] [ 'hmag' ] , thisfeatures [ 'info' ] [ 'kmag' ] )
            else :
                sdssr = np . nan
        else :
            sdssr = np . nan
        # the MAD of the light curve
        if ( magcol in thisfeatures and thisfeatures [ magcol ] and thisfeatures [ magcol ] [ 'mad' ] ) :
            lcmad = thisfeatures [ magcol ] [ 'mad' ]
        else :
            lcmad = np . nan
        # stetson index
        if ( magcol in thisfeatures and thisfeatures [ magcol ] and thisfeatures [ magcol ] [ 'stetsonj' ] ) :
            stetsonj = thisfeatures [ magcol ] [ 'stetsonj' ]
        else :
            stetsonj = np . nan
        # IQR
        if ( magcol in thisfeatures and thisfeatures [ magcol ] and thisfeatures [ magcol ] [ 'mag_iqr' ] ) :
            iqr = thisfeatures [ magcol ] [ 'mag_iqr' ]
        else :
            iqr = np . nan
        # eta
        if ( magcol in thisfeatures and thisfeatures [ magcol ] and thisfeatures [ magcol ] [ 'eta_normal' ] ) :
            eta = thisfeatures [ magcol ] [ 'eta_normal' ]
        else :
            eta = np . nan
        allobjects [ magcol ] [ 'objectid' ] . append ( objectid )
        allobjects [ magcol ] [ 'sdssr' ] . append ( sdssr )
        allobjects [ magcol ] [ 'lcmad' ] . append ( lcmad )
        allobjects [ magcol ] [ 'stetsonj' ] . append ( stetsonj )
        allobjects [ magcol ] [ 'iqr' ] . append ( iqr )
        allobjects [ magcol ] [ 'eta' ] . append ( eta )
    # done with collection of info
    LOGINFO ( 'finding objects above thresholds per magbin...' )
    # turn the info into arrays
    allobjects [ magcol ] [ 'objectid' ] = np . ravel ( np . array ( allobjects [ magcol ] [ 'objectid' ] ) )
    allobjects [ magcol ] [ 'sdssr' ] = np . ravel ( np . array ( allobjects [ magcol ] [ 'sdssr' ] ) )
    allobjects [ magcol ] [ 'lcmad' ] = np . ravel ( np . array ( allobjects [ magcol ] [ 'lcmad' ] ) )
    allobjects [ magcol ] [ 'stetsonj' ] = np . ravel ( np . array ( allobjects [ magcol ] [ 'stetsonj' ] ) )
    allobjects [ magcol ] [ 'iqr' ] = np . ravel ( np . array ( allobjects [ magcol ] [ 'iqr' ] ) )
    allobjects [ magcol ] [ 'eta' ] = np . ravel ( np . array ( allobjects [ magcol ] [ 'eta' ] ) )
    # only get finite elements everywhere
    thisfinind = ( np . isfinite ( allobjects [ magcol ] [ 'sdssr' ] ) & np . isfinite ( allobjects [ magcol ] [ 'lcmad' ] ) & np . isfinite ( allobjects [ magcol ] [ 'stetsonj' ] ) & np . isfinite ( allobjects [ magcol ] [ 'iqr' ] ) & np . isfinite ( allobjects [ magcol ] [ 'eta' ] ) )
    allobjects [ magcol ] [ 'objectid' ] = allobjects [ magcol ] [ 'objectid' ] [ thisfinind ]
    allobjects [ magcol ] [ 'sdssr' ] = allobjects [ magcol ] [ 'sdssr' ] [ thisfinind ]
    allobjects [ magcol ] [ 'lcmad' ] = allobjects [ magcol ] [ 'lcmad' ] [ thisfinind ]
    allobjects [ magcol ] [ 'stetsonj' ] = allobjects [ magcol ] [ 'stetsonj' ] [ thisfinind ]
    allobjects [ magcol ] [ 'iqr' ] = allobjects [ magcol ] [ 'iqr' ] [ thisfinind ]
    allobjects [ magcol ] [ 'eta' ] = allobjects [ magcol ] [ 'eta' ] [ thisfinind ]
    # invert eta so we can threshold the same way as the others
    allobjects [ magcol ] [ 'inveta' ] = 1.0 / allobjects [ magcol ] [ 'eta' ]
    # do the thresholding by magnitude bin
    magbininds = np . digitize ( allobjects [ magcol ] [ 'sdssr' ] , magbins )
    binned_objectids = [ ]
    binned_sdssr = [ ]
    binned_sdssr_median = [ ]
    binned_lcmad = [ ]
    binned_stetsonj = [ ]
    binned_iqr = [ ]
    binned_inveta = [ ]
    binned_count = [ ]
    binned_objectids_thresh_stetsonj = [ ]
    binned_objectids_thresh_iqr = [ ]
    binned_objectids_thresh_inveta = [ ]
    binned_objectids_thresh_all = [ ]
    binned_lcmad_median = [ ]
    binned_lcmad_stdev = [ ]
    binned_stetsonj_median = [ ]
    binned_stetsonj_stdev = [ ]
    binned_inveta_median = [ ]
    binned_inveta_stdev = [ ]
    binned_iqr_median = [ ]
    binned_iqr_stdev = [ ]
    # go through all the mag bins and get the thresholds for J , inveta , IQR
    for mbinind , magi in zip ( np . unique ( magbininds ) , range ( len ( magbins ) - 1 ) ) :
        thisbinind = np . where ( magbininds == mbinind )
        thisbin_sdssr_median = ( magbins [ magi ] + magbins [ magi + 1 ] ) / 2.0
        binned_sdssr_median . append ( thisbin_sdssr_median )
        thisbin_objectids = allobjects [ magcol ] [ 'objectid' ] [ thisbinind ]
        thisbin_sdssr = allobjects [ magcol ] [ 'sdssr' ] [ thisbinind ]
        thisbin_lcmad = allobjects [ magcol ] [ 'lcmad' ] [ thisbinind ]
        thisbin_stetsonj = allobjects [ magcol ] [ 'stetsonj' ] [ thisbinind ]
        thisbin_iqr = allobjects [ magcol ] [ 'iqr' ] [ thisbinind ]
        thisbin_inveta = allobjects [ magcol ] [ 'inveta' ] [ thisbinind ]
        thisbin_count = thisbin_objectids . size
        if thisbin_count > 4 :
            thisbin_lcmad_median = np . median ( thisbin_lcmad )
            thisbin_lcmad_stdev = np . median ( np . abs ( thisbin_lcmad - thisbin_lcmad_median ) ) * 1.483
            binned_lcmad_median . append ( thisbin_lcmad_median )
            binned_lcmad_stdev . append ( thisbin_lcmad_stdev )
            thisbin_stetsonj_median = np . median ( thisbin_stetsonj )
            thisbin_stetsonj_stdev = np . median ( np . abs ( thisbin_stetsonj - thisbin_stetsonj_median ) ) * 1.483
            binned_stetsonj_median . append ( thisbin_stetsonj_median )
            binned_stetsonj_stdev . append ( thisbin_stetsonj_stdev )
            # now get the objects above the required stdev threshold
            if isinstance ( magcol_min_stetj_stdev , float ) :
                thisbin_objectids_thresh_stetsonj = thisbin_objectids [ thisbin_stetsonj > ( thisbin_stetsonj_median + magcol_min_stetj_stdev * thisbin_stetsonj_stdev ) ]
            elif ( isinstance ( magcol_min_stetj_stdev , np . ndarray ) or isinstance ( magcol_min_stetj_stdev , list ) ) :
                thisbin_min_stetj_stdev = magcol_min_stetj_stdev [ magi ]
                if not np . isfinite ( thisbin_min_stetj_stdev ) :
                    LOGWARNING ( 'provided threshold stetson J stdev ' 'for magbin: %.3f is nan, using 2.0' % thisbin_sdssr_median )
                    thisbin_min_stetj_stdev = 2.0
                    # update the input list / array as well , since we ' ll be
                    # saving it to the output dict and using it to plot the
                    # variability thresholds
                    magcol_min_stetj_stdev [ magi ] = 2.0
                thisbin_objectids_thresh_stetsonj = thisbin_objectids [ thisbin_stetsonj > ( thisbin_stetsonj_median + thisbin_min_stetj_stdev * thisbin_stetsonj_stdev ) ]
            thisbin_iqr_median = np . median ( thisbin_iqr )
            thisbin_iqr_stdev = np . median ( np . abs ( thisbin_iqr - thisbin_iqr_median ) ) * 1.483
            binned_iqr_median . append ( thisbin_iqr_median )
            binned_iqr_stdev . append ( thisbin_iqr_stdev )
            # get the objects above the required stdev threshold
            if isinstance ( magcol_min_iqr_stdev , float ) :
                thisbin_objectids_thresh_iqr = thisbin_objectids [ thisbin_iqr > ( thisbin_iqr_median + magcol_min_iqr_stdev * thisbin_iqr_stdev ) ]
            elif ( isinstance ( magcol_min_iqr_stdev , np . ndarray ) or isinstance ( magcol_min_iqr_stdev , list ) ) :
                thisbin_min_iqr_stdev = magcol_min_iqr_stdev [ magi ]
                if not np . isfinite ( thisbin_min_iqr_stdev ) :
                    LOGWARNING ( 'provided threshold IQR stdev ' 'for magbin: %.3f is nan, using 2.0' % thisbin_sdssr_median )
                    thisbin_min_iqr_stdev = 2.0
                    # update the input list / array as well , since we ' ll be
                    # saving it to the output dict and using it to plot the
                    # variability thresholds
                    magcol_min_iqr_stdev [ magi ] = 2.0
                thisbin_objectids_thresh_iqr = thisbin_objectids [ thisbin_iqr > ( thisbin_iqr_median + thisbin_min_iqr_stdev * thisbin_iqr_stdev ) ]
            thisbin_inveta_median = np . median ( thisbin_inveta )
            thisbin_inveta_stdev = np . median ( np . abs ( thisbin_inveta - thisbin_inveta_median ) ) * 1.483
            binned_inveta_median . append ( thisbin_inveta_median )
            binned_inveta_stdev . append ( thisbin_inveta_stdev )
            if isinstance ( magcol_min_inveta_stdev , float ) :
                thisbin_objectids_thresh_inveta = thisbin_objectids [ thisbin_inveta > ( thisbin_inveta_median + magcol_min_inveta_stdev * thisbin_inveta_stdev ) ]
            elif ( isinstance ( magcol_min_inveta_stdev , np . ndarray ) or isinstance ( magcol_min_inveta_stdev , list ) ) :
                thisbin_min_inveta_stdev = magcol_min_inveta_stdev [ magi ]
                if not np . isfinite ( thisbin_min_inveta_stdev ) :
                    LOGWARNING ( 'provided threshold inveta stdev ' 'for magbin: %.3f is nan, using 2.0' % thisbin_sdssr_median )
                    thisbin_min_inveta_stdev = 2.0
                    # update the input list / array as well , since we ' ll be
                    # saving it to the output dict and using it to plot the
                    # variability thresholds
                    magcol_min_inveta_stdev [ magi ] = 2.0
                thisbin_objectids_thresh_inveta = thisbin_objectids [ thisbin_inveta > ( thisbin_inveta_median + thisbin_min_inveta_stdev * thisbin_inveta_stdev ) ]
        else :
            thisbin_objectids_thresh_stetsonj = ( np . array ( [ ] , dtype = np . unicode_ ) )
            thisbin_objectids_thresh_iqr = ( np . array ( [ ] , dtype = np . unicode_ ) )
            thisbin_objectids_thresh_inveta = ( np . array ( [ ] , dtype = np . unicode_ ) )
        # done with check for enough objects in the bin
        # get the intersection of all threshold objects to get objects that
        # lie above the threshold for all variable indices
        thisbin_objectids_thresh_all = reduce ( np . intersect1d , ( thisbin_objectids_thresh_stetsonj , thisbin_objectids_thresh_iqr , thisbin_objectids_thresh_inveta ) )
        binned_objectids . append ( thisbin_objectids )
        binned_sdssr . append ( thisbin_sdssr )
        binned_lcmad . append ( thisbin_lcmad )
        binned_stetsonj . append ( thisbin_stetsonj )
        binned_iqr . append ( thisbin_iqr )
        binned_inveta . append ( thisbin_inveta )
        binned_count . append ( thisbin_objectids . size )
        binned_objectids_thresh_stetsonj . append ( thisbin_objectids_thresh_stetsonj )
        binned_objectids_thresh_iqr . append ( thisbin_objectids_thresh_iqr )
        binned_objectids_thresh_inveta . append ( thisbin_objectids_thresh_inveta )
        binned_objectids_thresh_all . append ( thisbin_objectids_thresh_all )
    # done with magbins
    # update the output dict for this magcol
    allobjects [ magcol ] [ 'magbins' ] = magbins
    allobjects [ magcol ] [ 'binned_objectids' ] = binned_objectids
    allobjects [ magcol ] [ 'binned_sdssr_median' ] = binned_sdssr_median
    allobjects [ magcol ] [ 'binned_sdssr' ] = binned_sdssr
    allobjects [ magcol ] [ 'binned_count' ] = binned_count
    allobjects [ magcol ] [ 'binned_lcmad' ] = binned_lcmad
    allobjects [ magcol ] [ 'binned_lcmad_median' ] = binned_lcmad_median
    allobjects [ magcol ] [ 'binned_lcmad_stdev' ] = binned_lcmad_stdev
    allobjects [ magcol ] [ 'binned_stetsonj' ] = binned_stetsonj
    allobjects [ magcol ] [ 'binned_stetsonj_median' ] = binned_stetsonj_median
    allobjects [ magcol ] [ 'binned_stetsonj_stdev' ] = binned_stetsonj_stdev
    allobjects [ magcol ] [ 'binned_iqr' ] = binned_iqr
    allobjects [ magcol ] [ 'binned_iqr_median' ] = binned_iqr_median
    allobjects [ magcol ] [ 'binned_iqr_stdev' ] = binned_iqr_stdev
    allobjects [ magcol ] [ 'binned_inveta' ] = binned_inveta
    allobjects [ magcol ] [ 'binned_inveta_median' ] = binned_inveta_median
    allobjects [ magcol ] [ 'binned_inveta_stdev' ] = binned_inveta_stdev
    allobjects [ magcol ] [ 'binned_objectids_thresh_stetsonj' ] = ( binned_objectids_thresh_stetsonj )
    allobjects [ magcol ] [ 'binned_objectids_thresh_iqr' ] = ( binned_objectids_thresh_iqr )
    allobjects [ magcol ] [ 'binned_objectids_thresh_inveta' ] = ( binned_objectids_thresh_inveta )
    allobjects [ magcol ] [ 'binned_objectids_thresh_all' ] = ( binned_objectids_thresh_all )
    # get the common selected objects thru all measures
    try :
        allobjects [ magcol ] [ 'objectids_all_thresh_all_magbins' ] = np . unique ( np . concatenate ( allobjects [ magcol ] [ 'binned_objectids_thresh_all' ] ) )
    except ValueError :
        LOGWARNING ( 'not enough variable objects matching all thresholds' )
        allobjects [ magcol ] [ 'objectids_all_thresh_all_magbins' ] = ( np . array ( [ ] ) )
    allobjects [ magcol ] [ 'objectids_stetsonj_thresh_all_magbins' ] = np . unique ( np . concatenate ( allobjects [ magcol ] [ 'binned_objectids_thresh_stetsonj' ] ) )
    allobjects [ magcol ] [ 'objectids_inveta_thresh_all_magbins' ] = np . unique ( np . concatenate ( allobjects [ magcol ] [ 'binned_objectids_thresh_inveta' ] ) )
    allobjects [ magcol ] [ 'objectids_iqr_thresh_all_magbins' ] = np . unique ( np . concatenate ( allobjects [ magcol ] [ 'binned_objectids_thresh_iqr' ] ) )
    # turn these into np . arrays for easier plotting if they ' re lists
    if isinstance ( min_stetj_stdev , list ) :
        allobjects [ magcol ] [ 'min_stetj_stdev' ] = np . array ( magcol_min_stetj_stdev )
    else :
        allobjects [ magcol ] [ 'min_stetj_stdev' ] = magcol_min_stetj_stdev
    if isinstance ( min_iqr_stdev , list ) :
        allobjects [ magcol ] [ 'min_iqr_stdev' ] = np . array ( magcol_min_iqr_stdev )
    else :
        allobjects [ magcol ] [ 'min_iqr_stdev' ] = magcol_min_iqr_stdev
    if isinstance ( min_inveta_stdev , list ) :
        allobjects [ magcol ] [ 'min_inveta_stdev' ] = np . array ( magcol_min_inveta_stdev )
    else :
        allobjects [ magcol ] [ 'min_inveta_stdev' ] = magcol_min_inveta_stdev
    # this one doesn ' t get touched ( for now )
    allobjects [ magcol ] [ 'min_lcmad_stdev' ] = min_lcmad_stdev
# done with all magcols
allobjects [ 'magbins' ] = magbins
with open ( outfile , 'wb' ) as outfd :
    pickle . dump ( allobjects , outfd , protocol = pickle . HIGHEST_PROTOCOL )
return allobjects


signature:
def plot_variability_thresholds ( varthreshpkl , xmin_lcmad_stdev = 5.0 , xmin_stetj_stdev = 2.0 , xmin_iqr_stdev = 2.0 , xmin_inveta_stdev = 2.0 , lcformat = 'hat-sql' , lcformatdir = None , magcols = None ) :
    '''This makes plots for the variability threshold distributions .
 Parameters
 varthreshpkl : str
 The pickle produced by the function above .
 xmin _ lcmad _ stdev , xmin _ stetj _ stdev , xmin _ iqr _ stdev , xmin _ inveta _ stdev : float or np . array
 Values of the threshold values to override the ones in the
 ` vartresholdpkl ` . If provided , will plot the thresholds accordingly
 instead of using the ones in the input pickle directly .
 lcformat : str
 This is the ` formatkey ` associated with your light curve format , which
 you previously passed in to the ` lcproc . register _ lcformat `
 function . This will be used to look up how to find and read the light
 curves specified in ` basedir ` or ` use _ list _ of _ filenames ` .
 lcformatdir : str or None
 If this is provided , gives the path to a directory when you ' ve stored
 your lcformat description JSONs , other than the usual directories lcproc
 knows to search for them in . Use this along with ` lcformat ` to specify
 an LC format JSON file that ' s not currently registered with lcproc .
 magcols : list of str or None
 The magcol keys to use from the lcdict .
 Returns
 str
 The file name of the threshold plot generated .'''

implementation:
try :
    formatinfo = get_lcformat ( lcformat , use_lcformat_dir = lcformatdir )
    if formatinfo :
        ( dfileglob , readerfunc , dtimecols , dmagcols , derrcols , magsarefluxes , normfunc ) = formatinfo
    else :
        LOGERROR ( "can't figure out the light curve format" )
        return None
except Exception as e :
    LOGEXCEPTION ( "can't figure out the light curve format" )
    return None
if magcols is None :
    magcols = dmagcols
with open ( varthreshpkl , 'rb' ) as infd :
    allobjects = pickle . load ( infd )
magbins = allobjects [ 'magbins' ]
for magcol in magcols :
    min_lcmad_stdev = ( xmin_lcmad_stdev or allobjects [ magcol ] [ 'min_lcmad_stdev' ] )
    min_stetj_stdev = ( xmin_stetj_stdev or allobjects [ magcol ] [ 'min_stetj_stdev' ] )
    min_iqr_stdev = ( xmin_iqr_stdev or allobjects [ magcol ] [ 'min_iqr_stdev' ] )
    min_inveta_stdev = ( xmin_inveta_stdev or allobjects [ magcol ] [ 'min_inveta_stdev' ] )
    fig = plt . figure ( figsize = ( 20 , 16 ) )
    # the mag vs lcmad
    plt . subplot ( 221 )
    plt . plot ( allobjects [ magcol ] [ 'sdssr' ] , allobjects [ magcol ] [ 'lcmad' ] * 1.483 , marker = '.' , ms = 1.0 , linestyle = 'none' , rasterized = True )
    plt . plot ( allobjects [ magcol ] [ 'binned_sdssr_median' ] , np . array ( allobjects [ magcol ] [ 'binned_lcmad_median' ] ) * 1.483 , linewidth = 3.0 )
    plt . plot ( allobjects [ magcol ] [ 'binned_sdssr_median' ] , np . array ( allobjects [ magcol ] [ 'binned_lcmad_median' ] ) * 1.483 + min_lcmad_stdev * np . array ( allobjects [ magcol ] [ 'binned_lcmad_stdev' ] ) , linewidth = 3.0 , linestyle = 'dashed' )
    plt . xlim ( ( magbins . min ( ) - 0.25 , magbins . max ( ) ) )
    plt . xlabel ( 'SDSS r' )
    plt . ylabel ( r'lightcurve RMS (MAD $\times$ 1.483)' )
    plt . title ( '%s - SDSS r vs. light curve RMS' % magcol )
    plt . yscale ( 'log' )
    plt . tight_layout ( )
    # the mag vs stetsonj
    plt . subplot ( 222 )
    plt . plot ( allobjects [ magcol ] [ 'sdssr' ] , allobjects [ magcol ] [ 'stetsonj' ] , marker = '.' , ms = 1.0 , linestyle = 'none' , rasterized = True )
    plt . plot ( allobjects [ magcol ] [ 'binned_sdssr_median' ] , allobjects [ magcol ] [ 'binned_stetsonj_median' ] , linewidth = 3.0 )
    plt . plot ( allobjects [ magcol ] [ 'binned_sdssr_median' ] , np . array ( allobjects [ magcol ] [ 'binned_stetsonj_median' ] ) + min_stetj_stdev * np . array ( allobjects [ magcol ] [ 'binned_stetsonj_stdev' ] ) , linewidth = 3.0 , linestyle = 'dashed' )
    plt . xlim ( ( magbins . min ( ) - 0.25 , magbins . max ( ) ) )
    plt . xlabel ( 'SDSS r' )
    plt . ylabel ( 'Stetson J index' )
    plt . title ( '%s - SDSS r vs. Stetson J index' % magcol )
    plt . yscale ( 'log' )
    plt . tight_layout ( )
    # the mag vs IQR
    plt . subplot ( 223 )
    plt . plot ( allobjects [ magcol ] [ 'sdssr' ] , allobjects [ magcol ] [ 'iqr' ] , marker = '.' , ms = 1.0 , linestyle = 'none' , rasterized = True )
    plt . plot ( allobjects [ magcol ] [ 'binned_sdssr_median' ] , allobjects [ magcol ] [ 'binned_iqr_median' ] , linewidth = 3.0 )
    plt . plot ( allobjects [ magcol ] [ 'binned_sdssr_median' ] , np . array ( allobjects [ magcol ] [ 'binned_iqr_median' ] ) + min_iqr_stdev * np . array ( allobjects [ magcol ] [ 'binned_iqr_stdev' ] ) , linewidth = 3.0 , linestyle = 'dashed' )
    plt . xlabel ( 'SDSS r' )
    plt . ylabel ( 'IQR' )
    plt . title ( '%s - SDSS r vs. IQR' % magcol )
    plt . xlim ( ( magbins . min ( ) - 0.25 , magbins . max ( ) ) )
    plt . yscale ( 'log' )
    plt . tight_layout ( )
    # the mag vs IQR
    plt . subplot ( 224 )
    plt . plot ( allobjects [ magcol ] [ 'sdssr' ] , allobjects [ magcol ] [ 'inveta' ] , marker = '.' , ms = 1.0 , linestyle = 'none' , rasterized = True )
    plt . plot ( allobjects [ magcol ] [ 'binned_sdssr_median' ] , allobjects [ magcol ] [ 'binned_inveta_median' ] , linewidth = 3.0 )
    plt . plot ( allobjects [ magcol ] [ 'binned_sdssr_median' ] , np . array ( allobjects [ magcol ] [ 'binned_inveta_median' ] ) + min_inveta_stdev * np . array ( allobjects [ magcol ] [ 'binned_inveta_stdev' ] ) , linewidth = 3.0 , linestyle = 'dashed' )
    plt . xlabel ( 'SDSS r' )
    plt . ylabel ( r'$1/\eta$' )
    plt . title ( r'%s - SDSS r vs. $1/\eta$' % magcol )
    plt . xlim ( ( magbins . min ( ) - 0.25 , magbins . max ( ) ) )
    plt . yscale ( 'log' )
    plt . tight_layout ( )
    plt . savefig ( 'varfeatures-%s-%s-distributions.png' % ( varthreshpkl , magcol ) , bbox_inches = 'tight' )
    plt . close ( 'all' )


signature:
def get_stamp ( ra , decl , survey = 'DSS2 Red' , scaling = 'Linear' , sizepix = 300 , forcefetch = False , cachedir = '~/.astrobase/stamp-cache' , timeout = 10.0 , retry_failed = True , verbose = True , jitter = 5.0 ) :
    '''This gets a FITS cutout from the NASA GSFC SkyView service .
 This downloads stamps in FITS format from the NASA SkyView service :
 https : / / skyview . gsfc . nasa . gov / current / cgi / query . pl
 Parameters
 ra , decl : float
 These are decimal equatorial coordinates for the cutout center .
 survey : str
 The survey name to get the stamp from . This is one of the
 values in the ' SkyView Surveys ' option boxes on the SkyView
 webpage . Currently , we ' ve only tested using ' DSS2 Red ' as the value for
 this kwarg , but the other ones should work in principle .
 scaling : str
 This is the pixel value scaling function to use .
 sizepix : int
 The width and height of the cutout are specified by this value .
 forcefetch : bool
 If True , will disregard any existing cached copies of the stamp already
 downloaded corresponding to the requested center coordinates and
 redownload the FITS from the SkyView service .
 cachedir : str
 This is the path to the astrobase cache directory . All downloaded FITS
 stamps are stored here as . fits . gz files so we can immediately respond
 with the cached copy when a request is made for a coordinate center
 that ' s already been downloaded .
 timeout : float
 Sets the timeout in seconds to wait for a response from the NASA SkyView
 service .
 retry _ failed : bool
 If the initial request to SkyView fails , and this is True , will retry
 until it succeeds .
 verbose : bool
 If True , indicates progress .
 jitter : float
 This is used to control the scale of the random wait in seconds before
 starting the query . Useful in parallelized situations .
 Returns
 dict
 A dict of the following form is returned : :
 ' params ' : { input ra , decl and kwargs used } ,
 ' provenance ' : ' cached ' or ' new download ' ,
 ' fitsfile ' : FITS file to which the cutout was saved on disk'''

implementation:
# parse the given params into the correct format for the form
formposition = [ '%.4f, %.4f' % ( ra , decl ) ]
formscaling = [ scaling ]
formparams = copy . deepcopy ( SKYVIEW_PARAMS )
formparams [ 'Position' ] = formposition
formparams [ 'survey' ] [ 0 ] = survey
formparams [ 'scaling' ] = formscaling
formparams [ 'pixels' ] = [ '%s' % sizepix ]
# see if the cachedir exists
if '~' in cachedir :
    cachedir = os . path . expanduser ( cachedir )
if not os . path . exists ( cachedir ) :
    os . makedirs ( cachedir )
# figure out if we can get this image from the cache
cachekey = '%s-%s-%s-%s' % ( formposition [ 0 ] , survey , scaling , sizepix )
cachekey = hashlib . sha256 ( cachekey . encode ( ) ) . hexdigest ( )
cachefname = os . path . join ( cachedir , '%s.fits.gz' % cachekey )
provenance = 'cache'
# this is to handle older cached stamps that didn ' t include the sizepix
# parameter
if sizepix == 300 :
    oldcachekey = '%s-%s-%s' % ( formposition [ 0 ] , survey , scaling )
    oldcachekey = hashlib . sha256 ( oldcachekey . encode ( ) ) . hexdigest ( )
    oldcachefname = os . path . join ( cachedir , '%s.fits.gz' % oldcachekey )
    if os . path . exists ( oldcachefname ) :
        cachefname = oldcachefname
# if this exists in the cache and we ' re not refetching , get the frame
if forcefetch or ( not os . path . exists ( cachefname ) ) :
    provenance = 'new download'
    time . sleep ( random . randint ( 1 , jitter ) )
    # fire the request
    try :
        if verbose :
            LOGINFO ( 'submitting stamp request for %s, %s, %s, %s' % ( formposition [ 0 ] , survey , scaling , sizepix ) )
        req = requests . get ( SKYVIEW_URL , params = formparams , timeout = timeout )
        req . raise_for_status ( )
        # get the text of the response , this includes the locations of the
        # generated FITS on the server
        resp = req . text
        # find the URLS of the FITS
        fitsurls = FITS_REGEX . findall ( resp )
        # download the URLs
        if fitsurls :
            for fitsurl in fitsurls :
                fullfitsurl = urljoin ( FITS_BASEURL , fitsurl )
                if verbose :
                    LOGINFO ( 'getting %s' % fullfitsurl )
                fitsreq = requests . get ( fullfitsurl , timeout = timeout )
                with gzip . open ( cachefname , 'wb' ) as outfd :
                    outfd . write ( fitsreq . content )
        else :
            LOGERROR ( 'no FITS URLs found in query results for %s' % formposition )
            return None
    except requests . exceptions . HTTPError as e :
        LOGEXCEPTION ( 'SkyView stamp request for ' 'coordinates %s failed' % repr ( formposition ) )
        return None
    except requests . exceptions . Timeout as e :
        LOGERROR ( 'SkyView stamp request for ' 'coordinates %s did not complete within %s seconds' % ( repr ( formposition ) , timeout ) )
        return None
    except Exception as e :
        LOGEXCEPTION ( 'SkyView stamp request for ' 'coordinates %s failed' % repr ( formposition ) )
        return None
# DONE WITH FETCHING STUFF
# make sure the returned file is OK
try :
    stampfits = pyfits . open ( cachefname )
    stampfits . close ( )
    retdict = { 'params' : { 'ra' : ra , 'decl' : decl , 'survey' : survey , 'scaling' : scaling , 'sizepix' : sizepix } , 'provenance' : provenance , 'fitsfile' : cachefname }
    return retdict
except Exception as e :
    LOGERROR ( 'could not open cached FITS from Skyview download: %r' % { 'ra' : ra , 'decl' : decl , 'survey' : survey , 'scaling' : scaling , 'sizepix' : sizepix } )
    if retry_failed :
        return get_stamp ( ra , decl , survey = survey , scaling = scaling , sizepix = sizepix , forcefetch = True , cachedir = cachedir , timeout = timeout , verbose = verbose )
    else :
        return None


signature:
def _update_proxy ( self , change ) :
    """An observer which sends the state change to the proxy ."""

implementation:
if change [ 'type' ] == 'container' : # : Only update what ' s needed
    self . proxy . update_points ( change )
else :
    super ( MapPolyline , self ) . _update_proxy ( change )


implementation:
if change [ 'type' ] == 'container' : # : Only update what ' s needed
    self . proxy . update_points ( change )
else :
    super ( MapPolygon , self ) . _update_proxy ( change )


signature:
def handle_change ( self , change ) :
    """Handle changes from atom ContainerLists"""

implementation:
op = change [ 'operation' ]
if op in 'append' :
    self . add ( len ( change [ 'value' ] ) , LatLng ( * change [ 'item' ] ) )
elif op == 'insert' :
    self . add ( change [ 'index' ] , LatLng ( * change [ 'item' ] ) )
elif op == 'extend' :
    points = [ LatLng ( * p ) for p in change [ 'items' ] ]
    self . addAll ( [ bridge . encode ( c ) for c in points ] )
elif op == '__setitem__' :
    self . set ( change [ 'index' ] , LatLng ( * change [ 'newitem' ] ) )
elif op == 'pop' :
    self . remove ( change [ 'index' ] )
else :
    raise NotImplementedError ( "Unsupported change operation {}" . format ( op ) )


signature:
def create_widget ( self ) :
    """Create the underlying widget ."""

implementation:
self . init_options ( )
# : Retrieve the actual map
MapFragment . newInstance ( self . options ) . then ( self . on_map_fragment_created )
# Holder for the fragment
self . widget = FrameLayout ( self . get_context ( ) )
# I wrote this a few days ago and already forget how this hack works . . .
# lol We can ' t simply get a map reference using getMapAsync in the
# return value like we normally do with a normal call function return
# value . The bridge design was modified to store an object that cannot
# be decoded normally ( via a standard Bridge . Packer ) by saving the new
# object in the cache returning the id of the handler or proxy that
# invoked it . This way we can manually create a new id and pass that
# " future reference - able " object as our listener . At which point the
# bridge will create a reference entry in the cache for us with the of
# the object we gave it . Once in the cache we can use it like any
# bridge object we created .
self . map = GoogleMap ( __id__ = bridge . generate_id ( ) )


signature:
def init_options ( self ) :
    """Initialize the underlying map options ."""

implementation:
self . options = GoogleMapOptions ( )
d = self . declaration
self . set_map_type ( d . map_type )
if d . ambient_mode :
    self . set_ambient_mode ( d . ambient_mode )
if ( d . camera_position or d . camera_zoom or d . camera_tilt or d . camera_bearing ) :
    self . update_camera ( )
if d . map_bounds :
    self . set_map_bounds ( d . map_bounds )
if not d . show_compass :
    self . set_show_compass ( d . show_compass )
if not d . show_zoom_controls :
    self . set_show_zoom_controls ( d . show_zoom_controls )
if not d . show_toolbar :
    self . set_show_toolbar ( d . show_toolbar )
if d . lite_mode :
    self . set_lite_mode ( d . lite_mode )
if not d . rotate_gestures :
    self . set_rotate_gestures ( d . rotate_gestures )
if not d . scroll_gestures :
    self . set_scroll_gestures ( d . scroll_gestures )
if not d . tilt_gestures :
    self . set_tilt_gestures ( d . tilt_gestures )
if not d . zoom_gestures :
    self . set_zoom_gestures ( d . zoom_gestures )
if d . min_zoom :
    self . set_min_zoom ( d . min_zoom )
if d . max_zoom :
    self . set_max_zoom ( d . max_zoom )


signature:
def init_map ( self ) :
    """Add markers , polys , callouts , etc . ."""

implementation:
d = self . declaration
if d . show_location :
    self . set_show_location ( d . show_location )
if d . show_traffic :
    self . set_show_traffic ( d . show_traffic )
if d . show_indoors :
    self . set_show_indoors ( d . show_indoors )
if d . show_buildings :
    self . set_show_buildings ( d . show_buildings )
# : Local ref access is faster
mapview = self . map
mid = mapview . getId ( )
# : Connect signals
# : Camera
mapview . onCameraChange . connect ( self . on_camera_changed )
mapview . onCameraMoveStarted . connect ( self . on_camera_move_started )
mapview . onCameraMoveCanceled . connect ( self . on_camera_move_stopped )
mapview . onCameraIdle . connect ( self . on_camera_move_stopped )
mapview . setOnCameraChangeListener ( mid )
mapview . setOnCameraMoveStartedListener ( mid )
mapview . setOnCameraMoveCanceledListener ( mid )
mapview . setOnCameraIdleListener ( mid )
# : Clicks
mapview . onMapClick . connect ( self . on_map_clicked )
mapview . setOnMapClickListener ( mid )
mapview . onMapLongClick . connect ( self . on_map_long_clicked )
mapview . setOnMapLongClickListener ( mid )
# : Markers
mapview . onMarkerClick . connect ( self . on_marker_clicked )
mapview . setOnMarkerClickListener ( self . map . getId ( ) )
mapview . onMarkerDragStart . connect ( self . on_marker_drag_start )
mapview . onMarkerDrag . connect ( self . on_marker_drag )
mapview . onMarkerDragEnd . connect ( self . on_marker_drag_end )
mapview . setOnMarkerDragListener ( mid )
# : Info window
mapview . onInfoWindowClick . connect ( self . on_info_window_clicked )
mapview . onInfoWindowLongClick . connect ( self . on_info_window_long_clicked )
mapview . onInfoWindowClose . connect ( self . on_info_window_closed )
mapview . setOnInfoWindowClickListener ( mid )
mapview . setOnInfoWindowCloseListener ( mid )
mapview . setOnInfoWindowLongClickListener ( mid )
# : Polys
mapview . onPolygonClick . connect ( self . on_poly_clicked )
mapview . onPolylineClick . connect ( self . on_poly_clicked )
mapview . setOnPolygonClickListener ( mid )
mapview . setOnPolylineClickListener ( mid )
# : Circle
mapview . onCircleClick . connect ( self . on_circle_clicked )
mapview . setOnCircleClickListener ( mid )


signature:
def init_info_window_adapter ( self ) :
    """Initialize the info window adapter . Should only be done if one of
 the markers defines a custom view ."""

implementation:
adapter = self . adapter
if adapter :
    return
    # : Already initialized
adapter = GoogleMap . InfoWindowAdapter ( )
adapter . getInfoContents . connect ( self . on_info_window_contents_requested )
adapter . getInfoWindow . connect ( self . on_info_window_requested )
self . map . setInfoWindowAdapter ( adapter )


signature:
def on_map_fragment_created ( self , obj_id ) :
    """Create the fragment and pull the map reference when it ' s loaded ."""

implementation:
self . fragment = MapFragment ( __id__ = obj_id )
# : Setup callback so we know when the map is ready
self . map . onMapReady . connect ( self . on_map_ready )
self . fragment . getMapAsync ( self . map . getId ( ) )
context = self . get_context ( )
def on_transaction ( id ) :
    trans = FragmentTransaction ( __id__ = id )
    trans . add ( self . widget . getId ( ) , self . fragment )
    trans . commit ( )
def on_fragment_manager ( id ) :
    fm = FragmentManager ( __id__ = id )
    fm . beginTransaction ( ) . then ( on_transaction )
context . widget . getSupportFragmentManager ( ) . then ( on_fragment_manager )


signature:
def on_map_clicked ( self , pos ) :
    """Called when the map is clicked"""

implementation:
d = self . declaration
d . clicked ( { 'click' : 'short' , 'position' : tuple ( pos ) } )


signature:
def on_map_long_clicked ( self , pos ) :
    """Called when the map is clicked"""

implementation:
d = self . declaration
d . clicked ( { 'click' : 'long' , 'position' : tuple ( pos ) } )


signature:
def destroy ( self ) :
    """Remove the marker if it was added to the map when destroying"""

implementation:
marker = self . marker
parent = self . parent ( )
if marker :
    if parent :
        del parent . markers [ marker . __id__ ]
    marker . remove ( )
super ( AndroidMapItemBase , self ) . destroy ( )


signature:
def child_added ( self , child ) :
    """If a child is added we have to make sure the map adapter exists"""

implementation:
if child . widget : # TODO : Should we keep count and remove the adapter if not all
# markers request it ?
    self . parent ( ) . init_info_window_adapter ( )
super ( AndroidMapMarker , self ) . child_added ( child )


signature:
def on_marker ( self , marker ) :
    """Convert our options into the actual marker object"""

implementation:
mid , pos = marker
self . marker = Marker ( __id__ = mid )
mapview = self . parent ( )
# Save ref
mapview . markers [ mid ] = self
# Required so the packer can pass the id
self . marker . setTag ( mid )
# If we have a child widget we must configure the map to use the
# custom adapter
for w in self . child_widgets ( ) :
    mapview . init_info_window_adapter ( )
    break
d = self . declaration
if d . show_info :
    self . set_show_info ( d . show_info )
# : Can free the options now
del self . options


signature:
def on_marker ( self , mid ) :
    """Convert our options into the actual circle object"""

implementation:
self . marker = Circle ( __id__ = mid )
self . parent ( ) . markers [ mid ] = self
# : Required so the packer can pass the id
self . marker . setTag ( mid )
d = self . declaration
if d . clickable :
    self . set_clickable ( d . clickable )
# : Can free the options now
del self . options


signature:
def fit_transform ( self , raw_documents , y = None ) :
    """Learn the vocabulary dictionary and return term - document matrix .
 This is equivalent to fit followed by transform , but more efficiently
 implemented .
 Parameters
 raw _ documents : iterable
 An iterable which yields either str , unicode or file objects .
 Returns
 X : array , [ n _ samples , n _ features ]
 Document - term matrix ."""

implementation:
documents = super ( CountVectorizer , self ) . fit_transform ( raw_documents = raw_documents , y = y )
self . n = len ( raw_documents )
m = ( self . transform ( raw_documents ) > 0 ) . astype ( int )
m = m . sum ( axis = 0 ) . A1
self . period_ = m
self . df_ = m / self . n
return documents


signature:
def data ( self , X = None , y = None , sentences = None ) :
    """Add data to flow"""

implementation:
self . X = X
self . y = y
self . sentences = sentences


signature:
def transform ( self , transformer ) :
    """Add transformer to flow and apply transformer to data in flow
 Parameters
 transformer : Transformer
 a transformer to transform data"""

implementation:
self . transformers . append ( transformer )
from languageflow . transformer . tagged import TaggedTransformer
if isinstance ( transformer , TaggedTransformer ) :
    self . X , self . y = transformer . transform ( self . sentences )
if isinstance ( transformer , TfidfVectorizer ) :
    self . X = transformer . fit_transform ( self . X )
if isinstance ( transformer , CountVectorizer ) :
    self . X = transformer . fit_transform ( self . X )
if isinstance ( transformer , NumberRemover ) :
    self . X = transformer . transform ( self . X )
if isinstance ( transformer , MultiLabelBinarizer ) :
    self . y = transformer . fit_transform ( self . y )


signature:
def train ( self ) :
    """Train model with transformed data"""

implementation:
for i , model in enumerate ( self . models ) :
    N = [ int ( i * len ( self . y ) ) for i in self . lc_range ]
    for n in N :
        X = self . X [ : n ]
        y = self . y [ : n ]
        e = Experiment ( X , y , model . estimator , self . scores , self . validation_method )
        e . log_folder = self . log_folder
        e . train ( )


signature:
def export ( self , model_name , export_folder ) :
    """Export model and transformers to export _ folder
 Parameters
 model _ name : string
 name of model to export
 export _ folder : string
 folder to store exported model and transformers"""

implementation:
for transformer in self . transformers :
    if isinstance ( transformer , MultiLabelBinarizer ) :
        joblib . dump ( transformer , join ( export_folder , "label.transformer.bin" ) , protocol = 2 )
    if isinstance ( transformer , TfidfVectorizer ) :
        joblib . dump ( transformer , join ( export_folder , "tfidf.transformer.bin" ) , protocol = 2 )
    if isinstance ( transformer , CountVectorizer ) :
        joblib . dump ( transformer , join ( export_folder , "count.transformer.bin" ) , protocol = 2 )
    if isinstance ( transformer , NumberRemover ) :
        joblib . dump ( transformer , join ( export_folder , "number.transformer.bin" ) , protocol = 2 )
model = [ model for model in self . models if model . name == model_name ] [ 0 ]
e = Experiment ( self . X , self . y , model . estimator , None )
model_filename = join ( export_folder , "model.bin" )
e . export ( model_filename )


signature:
def fit_transform ( self , raw_documents , y = None ) :
    """Learn vocabulary and idf , return term - document matrix .
 This is equivalent to fit followed by transform , but more efficiently
 implemented .
 Parameters
 raw _ documents : iterable
 an iterable which yields either str , unicode or file objects
 Returns
 X : sparse matrix , [ n _ samples , n _ features ]
 Tf - idf - weighted document - term matrix ."""

implementation:
documents = super ( TfidfVectorizer , self ) . fit_transform ( raw_documents = raw_documents , y = y )
count = CountVectorizer ( encoding = self . encoding , decode_error = self . decode_error , strip_accents = self . strip_accents , lowercase = self . lowercase , preprocessor = self . preprocessor , tokenizer = self . tokenizer , stop_words = self . stop_words , token_pattern = self . token_pattern , ngram_range = self . ngram_range , analyzer = self . analyzer , max_df = self . max_df , min_df = self . min_df , max_features = self . max_features , vocabulary = self . vocabulary_ , binary = self . binary , dtype = self . dtype )
count . fit_transform ( raw_documents = raw_documents , y = y )
self . period_ = count . period_
self . df_ = count . df_
self . n = count . n
return documents


signature:
def fit ( self , X , y , coef_init = None , intercept_init = None , sample_weight = None ) :
    """Fit linear model with Stochastic Gradient Descent .
 Parameters
 X : { array - like , sparse matrix } , shape ( n _ samples , n _ features )
 Training data
 y : numpy array , shape ( n _ samples , )
 Target values
 coef _ init : array , shape ( n _ classes , n _ features )
 The initial coefficients to warm - start the optimization .
 intercept _ init : array , shape ( n _ classes , )
 The initial intercept to warm - start the optimization .
 sample _ weight : array - like , shape ( n _ samples , ) , optional
 Weights applied to individual samples .
 If not provided , uniform weights are assumed . These weights will
 be multiplied with class _ weight ( passed through the
 constructor ) if class _ weight is specified
 Returns
 self : returns an instance of self ."""

implementation: super ( SGDClassifier , self ) . fit ( X , y , coef_init , intercept_init , sample_weight )


signature:
def print_cm ( cm , labels , hide_zeroes = False , hide_diagonal = False , hide_threshold = None ) :
    """pretty print for confusion matrixes"""

implementation:
columnwidth = max ( [ len ( x ) for x in labels ] + [ 5 ] )
# 5 is value length
empty_cell = " " * columnwidth
# Print header
print ( "    " + empty_cell , end = " " )
for label in labels :
    print ( "%{0}s" . format ( columnwidth ) % label , end = " " )
print ( )
# Print rows
for i , label1 in enumerate ( labels ) :
    print ( "    %{0}s" . format ( columnwidth ) % label1 , end = " " )
    for j in range ( len ( labels ) ) :
        cell = "%{0}.1f" . format ( columnwidth ) % cm [ i , j ]
        if hide_zeroes :
            cell = cell if float ( cm [ i , j ] ) != 0 else empty_cell
        if hide_diagonal :
            cell = cell if i != j else empty_cell
        if hide_threshold :
            cell = cell if cm [ i , j ] > hide_threshold else empty_cell
        print ( cell , end = " " )
    print ( )


signature:
def load_big_file ( f ) :
    """Workaround for loading a big pickle file . Files over 2GB cause pickle errors on certin Mac and Windows distributions .
 : param f :
 : return :"""

implementation:
logger . info ( f'loading file {f}' )
with open ( f , 'r+b' ) as f_in : # mmap seems to be much more memory efficient
    bf = mmap . mmap ( f_in . fileno ( ) , 0 )
    f_in . close ( )
return bf


signature:
def url_to_filename ( url : str , etag : str = None ) -> str :
    """Converts a url into a filename in a reversible way .
 If ` etag ` is specified , add it on the end , separated by a period
 ( which necessarily won ' t appear in the base64 - encoded filename ) .
 Get rid of the quotes in the etag , since Windows doesn ' t like them ."""

implementation:
url_bytes = url . encode ( 'utf-8' )
b64_bytes = base64 . b64encode ( url_bytes )
decoded = b64_bytes . decode ( 'utf-8' )
if etag : # Remove quotes from etag
    etag = etag . replace ( '"' , '' )
    return f"{decoded}.{etag}"
else :
    return decoded


signature:
def filename_to_url ( filename : str ) -> Tuple [ str , str ] :
    """Recovers the the url from the encoded filename . Returns it and the ETag
 ( which may be ` ` None ` ` )"""

implementation:
try : # If there is an etag , it ' s everything after the first period
    decoded , etag = filename . split ( "." , 1 )
except ValueError : # Otherwise , use None
    decoded , etag = filename , None
filename_bytes = decoded . encode ( 'utf-8' )
url_bytes = base64 . b64decode ( filename_bytes )
return url_bytes . decode ( 'utf-8' ) , etag


signature:
def cached_path ( url_or_filename : str , cache_dir : Path ) -> Path :
    """Given something that might be a URL ( or might be a local path ) ,
 determine which . If it ' s a URL , download the file and cache it , and
 return the path to the cached file . If it ' s already a local path ,
 make sure the file exists and then return the path ."""

implementation:
dataset_cache = Path ( CACHE_ROOT ) / cache_dir
parsed = urlparse ( url_or_filename )
if parsed . scheme in ( 'http' , 'https' ) : # URL , so get it from the cache ( downloading if necessary )
    return get_from_cache ( url_or_filename , dataset_cache )
elif parsed . scheme == '' and Path ( url_or_filename ) . exists ( ) : # File , and it exists .
    return Path ( url_or_filename )
elif parsed . scheme == '' : # File , but it doesn ' t exist .
    raise FileNotFoundError ( "file {} not found" . format ( url_or_filename ) )
else : # Something unknown
    raise ValueError ( "unable to parse {} as a URL or as a local path" . format ( url_or_filename ) )


signature:
def get_from_cache ( url : str , cache_dir : Path = None ) -> Path :
    """Given a URL , look for the corresponding dataset in the local cache .
 If it ' s not there , download it . Then return the path to the cached file ."""

implementation:
cache_dir . mkdir ( parents = True , exist_ok = True )
filename = re . sub ( r'.+/' , '' , url )
# get cache path to put the file
cache_path = cache_dir / filename
if cache_path . exists ( ) :
    return cache_path
# make HEAD request to check ETag
response = requests . head ( url )
if response . status_code != 200 :
    if "www.dropbox.com" in url : # dropbox return code 301 , so we ignore this error
        pass
    else :
        raise IOError ( "HEAD request failed for url {}" . format ( url ) )
# add ETag to filename if it exists
# etag = response . headers . get ( " ETag " )
if not cache_path . exists ( ) : # Download to temporary file , then copy to cache dir once finished .
# Otherwise you get corrupt cache entries if the download gets interrupted .
    fd , temp_filename = tempfile . mkstemp ( )
    logger . info ( "%s not found in cache, downloading to %s" , url , temp_filename )
    # GET file object
    req = requests . get ( url , stream = True )
    content_length = req . headers . get ( 'Content-Length' )
    total = int ( content_length ) if content_length is not None else None
    progress = Tqdm . tqdm ( unit = "B" , total = total )
    with open ( temp_filename , 'wb' ) as temp_file :
        for chunk in req . iter_content ( chunk_size = 1024 ) :
            if chunk : # filter out keep - alive new chunks
                progress . update ( len ( chunk ) )
                temp_file . write ( chunk )
    progress . close ( )
    logger . info ( "copying %s to cache at %s" , temp_filename , cache_path )
    shutil . copyfile ( temp_filename , str ( cache_path ) )
    logger . info ( "removing temp file %s" , temp_filename )
    os . close ( fd )
    os . remove ( temp_filename )
return cache_path


signature:
def fit ( self , X , y ) :
    """Fit CRF according to X , y
 Parameters
 X : list of text
 each item is a text
 y : list
 each item is either a label ( in multi class problem ) or list of
 labels ( in multi label problem )"""

implementation:
trainer = pycrfsuite . Trainer ( verbose = True )
for xseq , yseq in zip ( X , y ) :
    trainer . append ( xseq , yseq )
trainer . set_params ( self . params )
if self . filename :
    filename = self . filename
else :
    filename = 'model.tmp'
trainer . train ( filename )
tagger = pycrfsuite . Tagger ( )
tagger . open ( filename )
self . estimator = tagger


signature:
def predict ( self , X ) :
    """Predict class labels for samples in X .
 Parameters
 X : { array - like , sparse matrix } , shape = [ n _ samples , n _ features ]
 Samples ."""

implementation:
if isinstance ( X [ 0 ] , list ) :
    return [ self . estimator . tag ( x ) for x in X ]
return self . estimator . tag ( X )


signature:
def serve ( self , port = 62000 ) :
    """Start LanguageBoard web application
 Parameters
 port : int
 port to serve web application"""

implementation:
from http . server import HTTPServer , CGIHTTPRequestHandler
os . chdir ( self . log_folder )
httpd = HTTPServer ( ( '' , port ) , CGIHTTPRequestHandler )
print ( "Starting LanguageBoard on port: " + str ( httpd . server_port ) )
webbrowser . open ( 'http://0.0.0.0:{}' . format ( port ) )
httpd . serve_forever ( )


signature:
def analyze ( self , output_folder = "." , auto_remove = False ) :
    """: type auto _ remove : boolean
 : param boolean auto _ remove : auto remove previous files in analyze folder"""

implementation:
if auto_remove :
    try :
        shutil . rmtree ( output_folder )
    except :
        pass
try :
    mkdir ( output_folder )
except :
    pass
tokens = [ token for sublist in self . sentences for token in sublist ]
df = pd . DataFrame ( tokens )
log = u""
log += u"Sentences    : {}\n" . format ( len ( self . sentences ) )
n = df . shape [ 1 ]
log += self . _analyze_first_token ( df , 0 , output_folder )
for i in range ( 1 , n ) :
    log += self . _analyze_field ( df , i , output_folder )
print ( log )
stat_file = join ( output_folder , "stats.txt" )
write ( stat_file , log )


signature:
def predict ( self , X ) :
    """In order to obtain the most likely label for a list of text
 Parameters
 X : list of string
 Raw texts
 Returns
 C : list of string
 List labels"""

implementation:
x = X
if not isinstance ( X , list ) :
    x = [ X ]
y = self . estimator . predict ( x )
y = [ item [ 0 ] for item in y ]
y = [ self . _remove_prefix ( label ) for label in y ]
if not isinstance ( X , list ) :
    y = y [ 0 ]
return y


signature:
def log ( model_folder , binary_file = "count.transformer.bin" , log_folder = "analyze" ) :
    """Parameters
 model _ folder : string
 folder contains binaries file of model
 binary _ file : string
 file path to count transformer binary file
 log _ folder : string
 log folder"""

implementation:
file = join ( model_folder , binary_file )
vectorizer = joblib . load ( file )
output = [ ]
for token in vectorizer . vocabulary_ :
    index = vectorizer . vocabulary_ [ token ]
    ngram = len ( token . split ( " " ) )
    output . append ( { "token" : token , "ngram" : ngram , "period" : vectorizer . period_ [ index ] . item ( ) , "df" : vectorizer . df_ [ index ] , } )
output = sorted ( output , key = lambda item : item [ "df" ] )
content = json . dumps ( output , ensure_ascii = False )
write ( join ( log_folder , "count.json" ) , content )


signature:
def fit ( self , X , y ) :
    """Fit KimCNNClassifier according to X , y
 Parameters
 X : list of string
 each item is a raw text
 y : list of string
 each item is a label"""

implementation:
# Data Loader
word_vector_transformer = WordVectorTransformer ( padding = 'max' )
X = word_vector_transformer . fit_transform ( X )
X = LongTensor ( X )
self . word_vector_transformer = word_vector_transformer
y_transformer = LabelEncoder ( )
y = y_transformer . fit_transform ( y )
y = torch . from_numpy ( y )
self . y_transformer = y_transformer
dataset = CategorizedDataset ( X , y )
dataloader = DataLoader ( dataset , batch_size = self . batch_size , shuffle = True , num_workers = 4 )
# Model
KERNEL_SIZES = self . kernel_sizes
NUM_KERNEL = self . num_kernel
EMBEDDING_DIM = self . embedding_dim
model = TextCNN ( vocab_size = word_vector_transformer . get_vocab_size ( ) , embedding_dim = EMBEDDING_DIM , output_size = len ( self . y_transformer . classes_ ) , kernel_sizes = KERNEL_SIZES , num_kernel = NUM_KERNEL )
if USE_CUDA :
    model = model . cuda ( )
# Train
EPOCH = self . epoch
LR = self . lr
loss_function = nn . CrossEntropyLoss ( )
optimizer = optim . Adam ( model . parameters ( ) , lr = LR )
for epoch in range ( EPOCH ) :
    losses = [ ]
    for i , data in enumerate ( dataloader ) :
        X , y = data
        X , y = Variable ( X ) , Variable ( y )
        optimizer . zero_grad ( )
        model . train ( )
        output = model ( X )
        loss = loss_function ( output , y )
        losses . append ( loss . data . tolist ( ) [ 0 ] )
        loss . backward ( )
        optimizer . step ( )
        if i % 100 == 0 :
            print ( "[%d/%d] mean_loss : %0.2f" % ( epoch , EPOCH , np . mean ( losses ) ) )
            losses = [ ]
self . model = model


signature:
def config_sources ( app , environment , cluster , configs_dirs , app_dir , local = False , build = False ) :
    """Return the config files for an environment & cluster specific app ."""

implementation:
sources = [ # Machine - specific
( configs_dirs , 'hostname' ) , ( configs_dirs , 'hostname-local' ) , ( configs_dirs , 'hostname-build' ) , # Global
( configs_dirs , 'common' ) , # Environment + Cluster
( configs_dirs , 'common-%s' % environment ) , ( configs_dirs , 'common-%s-%s' % ( environment , cluster ) ) , ( configs_dirs , 'common-local' ) , ( configs_dirs , 'common-build' ) , # Machine - specific overrides
( configs_dirs , 'common-overrides' ) , # Application - specific
( [ app_dir ] , '%s-default' % app ) , ( [ app_dir ] , '%s-%s' % ( app , environment ) ) , ( [ app_dir ] , '%s-%s-%s' % ( app , environment , cluster ) ) , ( configs_dirs , app ) , ( configs_dirs , '%s-%s' % ( app , environment ) ) , ( configs_dirs , '%s-%s-%s' % ( app , environment , cluster ) ) , ( [ app_dir ] , '%s-local' % app ) , ( [ app_dir ] , '%s-build' % app ) , ( configs_dirs , '%s-local' % app ) , ( configs_dirs , '%s-build' % app ) , # Machine - specific application override
( configs_dirs , '%s-overrides' % app ) , ]
# Filter out build sources if not requested
if not build :
    sources = [ source for source in sources if not source [ 1 ] . endswith ( '-build' ) ]
    # Filter out local sources if not build and not local
    if not local :
        sources = [ source for source in sources if not source [ 1 ] . endswith ( '-local' ) ]
return available_sources ( sources )


signature:
def available_sources ( sources ) :
    """Yield the sources that are present ."""

implementation:
for dirs , name in sources :
    for directory in dirs :
        fn = os . path . join ( directory , name ) + '.py'
        if os . path . isfile ( fn ) :
            yield fn


signature:
def smush_config ( sources , initial = None ) :
    """Merge the configuration sources and return the resulting DotDict ."""

implementation:
if initial is None :
    initial = { }
config = DotDict ( initial )
for fn in sources :
    log . debug ( 'Merging %s' , fn )
    mod = get_config_module ( fn )
    config = mod . update ( config )
    log . debug ( 'Current config:\n%s' , json . dumps ( config , indent = 4 , cls = LenientJSONEncoder ) )
return config


signature:
def merge_dicts ( d1 , d2 , _path = None ) :
    """Merge dictionary d2 into d1 , overriding entries in d1 with values from d2.
 d1 is mutated .
 _ path is for internal , recursive use ."""

implementation:
if _path is None :
    _path = ( )
if isinstance ( d1 , dict ) and isinstance ( d2 , dict ) :
    for k , v in d2 . items ( ) :
        if isinstance ( v , MissingValue ) and v . name is None :
            v . name = '.' . join ( _path + ( k , ) )
        if isinstance ( v , DeletedValue ) :
            d1 . pop ( k , None )
        elif k not in d1 :
            if isinstance ( v , dict ) :
                d1 [ k ] = merge_dicts ( { } , v , _path + ( k , ) )
            else :
                d1 [ k ] = v
        else :
            if isinstance ( d1 [ k ] , dict ) and isinstance ( v , dict ) :
                d1 [ k ] = merge_dicts ( d1 [ k ] , v , _path + ( k , ) )
            elif isinstance ( d1 [ k ] , list ) and isinstance ( v , list ) : # Lists are only supported as leaves
                d1 [ k ] += v
            elif isinstance ( d1 [ k ] , MissingValue ) :
                d1 [ k ] = v
            elif d1 [ k ] is None :
                d1 [ k ] = v
            elif type ( d1 [ k ] ) == type ( v ) :
                d1 [ k ] = v
            else :
                raise TypeError ( 'Refusing to replace a %s with a %s' % ( type ( d1 [ k ] ) , type ( v ) ) )
else :
    raise TypeError ( 'Cannot merge a %s with a %s' % ( type ( d1 ) , type ( d2 ) ) )
return d1


signature:
def filter_dict ( unfiltered , filter_keys ) :
    """Return a subset of a dictionary using the specified keys ."""

implementation:
filtered = DotDict ( )
for k in filter_keys :
    filtered [ k ] = unfiltered [ k ]
return filtered


signature:
def _convert_item ( self , obj ) :
    """Convert obj into a DotDict , or list of DotDict .
 Directly nested lists aren ' t supported .
 Returns the result"""

implementation:
if isinstance ( obj , dict ) and not isinstance ( obj , DotDict ) :
    obj = DotDict ( obj )
elif isinstance ( obj , list ) : # must mutate and not just reassign , otherwise it will
# just use original object mutable / immutable
    for i , item in enumerate ( obj ) :
        if isinstance ( item , dict ) and not isinstance ( item , DotDict ) :
            obj [ i ] = DotDict ( item )
return obj


signature:
def filter_config ( config , deploy_config ) :
    """Return a config subset using the filter defined in the deploy config ."""

implementation:
if not os . path . isfile ( deploy_config ) :
    return DotDict ( )
config_module = get_config_module ( deploy_config )
return config_module . filter ( config )


signature:
def seeded_auth_token ( client , service , seed ) :
    """Return an auth token based on the client + service + seed tuple ."""

implementation:
hash_func = hashlib . md5 ( )
token = ',' . join ( ( client , service , seed ) ) . encode ( 'utf-8' )
hash_func . update ( token )
return hash_func . hexdigest ( )


signature:
def write_config ( config , app_dir , filename = 'configuration.json' ) :
    """Write configuration to the applicaiton directory ."""

implementation:
path = os . path . join ( app_dir , filename )
with open ( path , 'w' ) as f :
    json . dump ( config , f , indent = 4 , cls = DetectMissingEncoder , separators = ( ',' , ': ' ) )


signature:
def get_config_module ( config_pathname ) :
    """Imports the config file to yoconfigurator . configs . < config _ basename > ."""

implementation:
configs_mod = 'yoconfigurator.configs'
if configs_mod not in sys . modules :
    sys . modules [ configs_mod ] = types . ModuleType ( configs_mod )
module_name = os . path . basename ( config_pathname ) . rsplit ( '.' , 1 ) [ 0 ]
module_name = configs_mod + '.' + module_name
return _load_module ( module_name , config_pathname )


signature:
def validate_date ( date_text ) :
    """Return True if valid , raise ValueError if not"""

implementation:
try :
    if int ( date_text ) < 0 :
        return True
except ValueError :
    pass
try :
    datetime . strptime ( date_text , '%Y-%m-%d' )
    return True
except ValueError :
    pass
raise ValueError ( 'Dates must be negative integers or YYYY-MM-DD in the past.' )


signature:
def get_download_total ( rows ) :
    """Return the total downloads , and the downloads column"""

implementation:
headers = rows . pop ( 0 )
index = headers . index ( 'download_count' )
total_downloads = sum ( int ( row [ index ] ) for row in rows )
rows . insert ( 0 , headers )
return total_downloads , index


signature:
def add_download_total ( rows ) :
    """Add a final row to rows showing the total downloads"""

implementation:
total_row = [ "" ] * len ( rows [ 0 ] )
total_row [ 0 ] = "Total"
total_downloads , downloads_column = get_download_total ( rows )
total_row [ downloads_column ] = str ( total_downloads )
rows . append ( total_row )
return rows


signature:
def pypinfo ( ctx , project , fields , auth , run , json , indent , timeout , limit , days , start_date , end_date , where , order , all_installers , percent , markdown , ) :
    """Valid fields are : \n
 project | version | file | pyversion | percent3 | percent2 | impl | impl - version | \n
 openssl | date | month | year | country | installer | installer - version | \n
 setuptools - version | system | system - release | distro | distro - version | cpu"""

implementation:
if auth :
    set_credentials ( auth )
    click . echo ( 'Credentials location set to "{}".' . format ( get_credentials ( ) ) )
    return
if project is None and not fields :
    click . echo ( ctx . get_help ( ) )
    return
parsed_fields = [ ]
for field in fields :
    parsed = FIELD_MAP . get ( field )
    if parsed is None :
        raise ValueError ( '"{}" is an unsupported field.' . format ( field ) )
    parsed_fields . append ( parsed )
order_name = order
order = FIELD_MAP . get ( order )
if order :
    order_name = order . name
    parsed_fields . insert ( 0 , order )
built_query = build_query ( project , parsed_fields , limit = limit , days = days , start_date = start_date , end_date = end_date , where = where , order = order_name , pip = not all_installers , )
if run :
    client = create_client ( get_credentials ( ) )
    query_job = client . query ( built_query , job_config = create_config ( ) )
    query_rows = query_job . result ( timeout = timeout // 1000 )
    # Cached
    from_cache = not not query_job . cache_hit
    # Processed
    bytes_processed = query_job . total_bytes_processed or 0
    processed_amount , processed_unit = convert_units ( bytes_processed )
    # Billed
    bytes_billed = query_job . total_bytes_billed or 0
    billed_amount , billed_unit = convert_units ( bytes_billed )
    # Cost
    billing_tier = query_job . billing_tier or 1
    estimated_cost = Decimal ( TIER_COST * billing_tier ) / TB * Decimal ( bytes_billed )
    estimated_cost = str ( estimated_cost . quantize ( TO_CENTS , rounding = ROUND_UP ) )
    rows = parse_query_result ( query_job , query_rows )
    if percent :
        rows = add_percentages ( rows , include_sign = not json )
    # Only for tables , and if more than the header row + a single data row
    if len ( rows ) > 2 and not json :
        rows = add_download_total ( rows )
    if not json :
        click . echo ( 'Served from cache: {}' . format ( from_cache ) )
        click . echo ( 'Data processed: {:.2f} {}' . format ( processed_amount , processed_unit ) )
        click . echo ( 'Data billed: {:.2f} {}' . format ( billed_amount , billed_unit ) )
        click . echo ( 'Estimated cost: ${}' . format ( estimated_cost ) )
        click . echo ( )
        click . echo ( tabulate ( rows , markdown ) )
    else :
        query_info = { 'cached' : from_cache , 'bytes_processed' : bytes_processed , 'bytes_billed' : bytes_billed , 'estimated_cost' : estimated_cost , }
        click . echo ( format_json ( rows , query_info , indent ) )
else :
    click . echo ( built_query )


signature:
def find_and_patch_entry ( soup , entry ) :
    """Modify soup so Dash . app can generate TOCs on the fly ."""

implementation:
link = soup . find ( "a" , { "class" : "headerlink" } , href = "#" + entry . anchor )
tag = soup . new_tag ( "a" )
tag [ "name" ] = APPLE_REF_TEMPLATE . format ( entry . type , entry . name )
if link :
    link . parent . insert ( 0 , tag )
    return True
elif entry . anchor . startswith ( "module-" ) :
    soup . h1 . parent . insert ( 0 , tag )
    return True
else :
    return False


signature:
def inv_entry_to_path ( data ) :
    """Determine the path from the intersphinx inventory entry
 Discard the anchors between head and tail to make it
 compatible with situations where extra meta information is encoded ."""

implementation:
path_tuple = data [ 2 ] . split ( "#" )
if len ( path_tuple ) > 1 :
    path_str = "#" . join ( ( path_tuple [ 0 ] , path_tuple [ - 1 ] ) )
else :
    path_str = data [ 2 ]
return path_str


signature:
def main ( source , force , name , quiet , verbose , destination , add_to_dash , add_to_global , icon , index_page , enable_js , online_redirect_url , parser , ) :
    """Convert docs from SOURCE to Dash . app ' s docset format ."""

implementation:
try :
    logging . config . dictConfig ( create_log_config ( verbose = verbose , quiet = quiet ) )
except ValueError as e :
    click . secho ( e . args [ 0 ] , fg = "red" )
    raise SystemExit ( 1 )
if icon :
    icon_data = icon . read ( )
    if not icon_data . startswith ( PNG_HEADER ) :
        log . error ( '"{}" is not a valid PNG image.' . format ( click . format_filename ( icon . name ) ) )
        raise SystemExit ( 1 )
else :
    icon_data = None
source , dest , name = setup_paths ( source , destination , name = name , add_to_global = add_to_global , force = force , )
if parser is None :
    parser = parsers . get_doctype ( source )
    if parser is None :
        log . error ( '"{}" does not contain a known documentation format.' . format ( click . format_filename ( source ) ) )
        raise SystemExit ( errno . EINVAL )
docset = prepare_docset ( source , dest , name , index_page , enable_js , online_redirect_url )
doc_parser = parser ( doc_path = docset . docs )
log . info ( ( "Converting " + click . style ( "{parser_name}" , bold = True ) + ' docs from "{src}" to "{dst}".' ) . format ( parser_name = parser . name , src = click . format_filename ( source , shorten = True ) , dst = click . format_filename ( dest ) , ) )
with docset . db_conn :
    log . info ( "Parsing documentation..." )
    toc = patch_anchors ( doc_parser , show_progressbar = not quiet )
    for entry in doc_parser . parse ( ) :
        docset . db_conn . execute ( "INSERT INTO searchIndex VALUES (NULL, ?, ?, ?)" , entry . as_tuple ( ) , )
        toc . send ( entry )
    count = docset . db_conn . execute ( "SELECT COUNT(1) FROM searchIndex" ) . fetchone ( ) [ 0 ]
    log . info ( ( "Added " + click . style ( "{count:,}" , fg = "green" if count > 0 else "red" ) + " index entries." ) . format ( count = count ) )
    toc . close ( )
if icon_data :
    add_icon ( icon_data , dest )
if add_to_dash or add_to_global :
    log . info ( "Adding to Dash.app..." )
    os . system ( 'open -a dash "{}"' . format ( dest ) )


signature:
def create_log_config ( verbose , quiet ) :
    """We use logging ' s levels as an easy - to - use verbosity controller ."""

implementation:
if verbose and quiet :
    raise ValueError ( "Supplying both --quiet and --verbose makes no sense." )
elif verbose :
    level = logging . DEBUG
elif quiet :
    level = logging . ERROR
else :
    level = logging . INFO
logger_cfg = { "handlers" : [ "click_handler" ] , "level" : level }
return { "version" : 1 , "formatters" : { "click_formatter" : { "format" : "%(message)s" } } , "handlers" : { "click_handler" : { "level" : level , "class" : "doc2dash.__main__.ClickEchoHandler" , "formatter" : "click_formatter" , } } , "loggers" : { "doc2dash" : logger_cfg , "__main__" : logger_cfg } , }


signature:
def setup_paths ( source , destination , name , add_to_global , force ) :
    """Determine source and destination using the options ."""

implementation:
if source [ - 1 ] == "/" :
    source = source [ : - 1 ]
if not name :
    name = os . path . split ( source ) [ - 1 ]
elif name . endswith ( ".docset" ) :
    name = name . replace ( ".docset" , "" )
if add_to_global :
    destination = DEFAULT_DOCSET_PATH
dest = os . path . join ( destination or "" , name + ".docset" )
dst_exists = os . path . lexists ( dest )
if dst_exists and force :
    shutil . rmtree ( dest )
elif dst_exists :
    log . error ( 'Destination path "{}" already exists.' . format ( click . format_filename ( dest ) ) )
    raise SystemExit ( errno . EEXIST )
return source , dest , name


signature:
def prepare_docset ( source , dest , name , index_page , enable_js , online_redirect_url ) :
    """Create boilerplate files & directories and copy vanilla docs inside .
 Return a tuple of path to resources and connection to sqlite db ."""

implementation:
resources = os . path . join ( dest , "Contents" , "Resources" )
docs = os . path . join ( resources , "Documents" )
os . makedirs ( resources )
db_conn = sqlite3 . connect ( os . path . join ( resources , "docSet.dsidx" ) )
db_conn . row_factory = sqlite3 . Row
db_conn . execute ( "CREATE TABLE searchIndex(id INTEGER PRIMARY KEY, name TEXT, " "type TEXT, path TEXT)" )
db_conn . commit ( )
plist_path = os . path . join ( dest , "Contents" , "Info.plist" )
plist_cfg = { "CFBundleIdentifier" : name , "CFBundleName" : name , "DocSetPlatformFamily" : name . lower ( ) , "DashDocSetFamily" : "python" , "isDashDocset" : True , "isJavaScriptEnabled" : enable_js , }
if index_page is not None :
    plist_cfg [ "dashIndexFilePath" ] = index_page
if online_redirect_url is not None :
    plist_cfg [ "DashDocSetFallbackURL" ] = online_redirect_url
write_plist ( plist_cfg , plist_path )
shutil . copytree ( source , docs )
return DocSet ( path = dest , docs = docs , plist = plist_path , db_conn = db_conn )


signature:
def add_icon ( icon_data , dest ) :
    """Add icon to docset"""

implementation:
with open ( os . path . join ( dest , "icon.png" ) , "wb" ) as f :
    f . write ( icon_data )


signature:
def patch_anchors ( parser , show_progressbar ) :
    """Consume ` ` ParseEntry ` ` s then patch docs for TOCs by calling
 * parser * ' s ` ` find _ and _ patch _ entry ` ` ."""

implementation:
files = defaultdict ( list )
try :
    while True :
        pentry = ( yield )
        try :
            fname , anchor = pentry . path . split ( "#" )
            files [ fname ] . append ( TOCEntry ( name = pentry . name , type = pentry . type , anchor = anchor ) )
        except ValueError : # pydoctor has no anchors for e . g . classes
            pass
except GeneratorExit :
    pass
def patch_files ( files ) :
    for fname , entries in files :
        full_path = os . path . join ( parser . doc_path , fname )
        with codecs . open ( full_path , mode = "r" , encoding = "utf-8" ) as fp :
            soup = BeautifulSoup ( fp , "html.parser" )
            for entry in entries :
                if not parser . find_and_patch_entry ( soup , entry ) :
                    log . debug ( "Can't find anchor {} in {}." . format ( entry . anchor , click . format_filename ( fname ) ) )
        with open ( full_path , mode = "wb" ) as fp :
            fp . write ( soup . encode ( "utf-8" ) )
if show_progressbar is True :
    with click . progressbar ( files . items ( ) , width = 0 , length = len ( files ) , label = "Adding table of contents meta data..." , ) as pbar :
        patch_files ( pbar )
else :
    patch_files ( files . items ( ) )


signature:
def has_file_with ( path , filename , content ) :
    """Check whether * filename * in * path * contains the string * content * ."""

implementation:
try :
    with open ( os . path . join ( path , filename ) , "rb" ) as f :
        return content in f . read ( )
except IOError as e :
    if e . errno == errno . ENOENT :
        return False
    else :
        raise


signature:
def run_cell ( self , cell ) :
    """Run the Cell code using the IPython globals and locals
 Args :
 cell ( str ) : Python code to be executed"""

implementation:
globals = self . ipy_shell . user_global_ns
locals = self . ipy_shell . user_ns
globals . update ( { "__ipy_scope__" : None , } )
try :
    with redirect_stdout ( self . stdout ) :
        self . run ( cell , globals , locals )
except :
    self . code_error = True
    if self . options . debug :
        raise BdbQuit
finally :
    self . finalize ( )


signature:
def get_stack_data ( self , frame , traceback , event_type ) :
    """Get the stack frames data at each of the hooks above ( Ie . for each
 line of the Python code )"""

implementation:
heap_data = Heap ( self . options )
stack_data = StackFrames ( self . options )
stack_frames , cur_frame_ind = self . get_stack ( frame , traceback )
for frame_ind , ( frame , lineno ) in enumerate ( stack_frames ) :
    skip_this_stack = False
    # Skip the self . run calling frame ( first frame )
    if frame_ind == 0 :
        continue
    # Skip stack after a certain stack frame depth
    if len ( stack_data ) > self . options . depth :
        skip_this_stack = True
        break
    # Skip stack when frames dont belong to the current notebook or
    # current cell , I . e . frames in another global scope altogether
    # or frames in other cells
    if ( not self . is_notebook_frame ( frame ) or self . is_other_cell_frame ( frame ) ) :
        if not self . options . step_all :
            skip_this_stack = True
            break
        lineno = 0
        # So line markers dont display for these frames
    else :
        lineno += 1
    # Because cell magic is actually line 1
    # Filter out ignored names from the frame locals
    user_locals = filter_dict ( frame . f_locals , ignore_vars + list ( self . ipy_shell . user_ns_hidden . keys ( ) ) )
    # Add frame and heap data
    stack_data . add ( frame , lineno , event_type , user_locals )
    heap_data . add ( user_locals )
if not skip_this_stack and not stack_data . is_empty ( ) :
    self . trace_history . append ( stack_data , heap_data , self . stdout . getvalue ( ) )


signature:
def filter_dict ( d , exclude ) :
    """Return a new dict with specified keys excluded from the origional dict
 Args :
 d ( dict ) : origional dict
 exclude ( list ) : The keys that are excluded"""

implementation:
ret = { }
for key , value in d . items ( ) :
    if key not in exclude :
        ret . update ( { key : value } )
return ret


signature:
def redirect_stdout ( new_stdout ) :
    """Redirect the stdout
 Args :
 new _ stdout ( io . StringIO ) : New stdout to use instead"""

implementation:
old_stdout , sys . stdout = sys . stdout , new_stdout
try :
    yield None
finally :
    sys . stdout = old_stdout


signature:
def format ( obj , options ) :
    """Return a string representation of the Python object
 Args :
 obj : The Python object
 options : Format options"""

implementation:
formatters = { float_types : lambda x : '{:.{}g}' . format ( x , options . digits ) , }
for _types , fmtr in formatters . items ( ) :
    if isinstance ( obj , _types ) :
        return fmtr ( obj )
try :
    if six . PY2 and isinstance ( obj , six . string_types ) :
        return str ( obj . encode ( 'utf-8' ) )
    return str ( obj )
except :
    return 'OBJECT'


signature:
def get_type_info ( obj ) :
    """Get type information for a Python object
 Args :
 obj : The Python object
 Returns :
 tuple : ( object type " catagory " , object type name )"""

implementation:
if isinstance ( obj , primitive_types ) :
    return ( 'primitive' , type ( obj ) . __name__ )
if isinstance ( obj , sequence_types ) :
    return ( 'sequence' , type ( obj ) . __name__ )
if isinstance ( obj , array_types ) :
    return ( 'array' , type ( obj ) . __name__ )
if isinstance ( obj , key_value_types ) :
    return ( 'key-value' , type ( obj ) . __name__ )
if isinstance ( obj , types . ModuleType ) :
    return ( 'module' , type ( obj ) . __name__ )
if isinstance ( obj , ( types . FunctionType , types . MethodType ) ) :
    return ( 'function' , type ( obj ) . __name__ )
if isinstance ( obj , type ) :
    if hasattr ( obj , '__dict__' ) :
        return ( 'class' , obj . __name__ )
if isinstance ( type ( obj ) , type ) :
    if hasattr ( obj , '__dict__' ) :
        cls_name = type ( obj ) . __name__
        if cls_name == 'classobj' :
            cls_name = obj . __name__
            return ( 'class' , '{}' . format ( cls_name ) )
        if cls_name == 'instance' :
            cls_name = obj . __class__ . __name__
        return ( 'instance' , '{} instance' . format ( cls_name ) )
return ( 'unknown' , type ( obj ) . __name__ )


signature:
def refresh ( self ) :
    """Reloads the wallet and its accounts . By default , this method is called only once ,
 on : class : ` Wallet ` initialization . When the wallet is accessed by multiple clients or
 exists in multiple instances , calling ` refresh ( ) ` will be necessary to update
 the list of accounts ."""

implementation:
self . accounts = self . accounts or [ ]
idx = 0
for _acc in self . _backend . accounts ( ) :
    _acc . wallet = self
    try :
        if self . accounts [ idx ] :
            continue
    except IndexError :
        pass
    self . accounts . append ( _acc )
    idx += 1


signature:
def spend_key ( self ) :
    """Returns private spend key . None if wallet is view - only .
 : rtype : str or None"""

implementation:
key = self . _backend . spend_key ( )
if key == numbers . EMPTY_KEY :
    return None
return key


signature:
def new_account ( self , label = None ) :
    """Creates new account , appends it to the : class : ` Wallet ` ' s account list and returns it .
 : param label : account label as ` str `
 : rtype : : class : ` Account `"""

implementation:
acc , addr = self . _backend . new_account ( label = label )
assert acc . index == len ( self . accounts )
self . accounts . append ( acc )
return acc


signature:
def confirmations ( self , txn_or_pmt ) :
    """Returns the number of confirmations for given
 : class : ` Transaction < monero . transaction . Transaction > ` or
 : class : ` Payment < monero . transaction . Payment > ` object .
 : rtype : int"""

implementation:
if isinstance ( txn_or_pmt , Payment ) :
    txn = txn_or_pmt . transaction
else :
    txn = txn_or_pmt
try :
    return max ( 0 , self . height ( ) - txn . height )
except TypeError :
    return 0


signature:
def get_address ( self , major , minor ) :
    """Calculates sub - address for account index ( ` major ` ) and address index within
 the account ( ` minor ` ) .
 : rtype : : class : ` BaseAddress < monero . address . BaseAddress > `"""

implementation:
# ensure indexes are within uint32
if major < 0 or major >= 2 ** 32 :
    raise ValueError ( 'major index {} is outside uint32 range' . format ( major ) )
if minor < 0 or minor >= 2 ** 32 :
    raise ValueError ( 'minor index {} is outside uint32 range' . format ( minor ) )
master_address = self . address ( )
if major == minor == 0 :
    return master_address
master_svk = unhexlify ( self . view_key ( ) )
master_psk = unhexlify ( self . address ( ) . spend_key ( ) )
# m = Hs ( " SubAddr \ 0 " | | master _ svk | | major | | minor )
hsdata = b'' . join ( [ b'SubAddr\0' , master_svk , struct . pack ( '<I' , major ) , struct . pack ( '<I' , minor ) ] )
m = keccak_256 ( hsdata ) . digest ( )
# D = master _ psk + m * B
D = ed25519 . add_compressed ( ed25519 . decodepoint ( master_psk ) , ed25519 . scalarmult ( ed25519 . B , ed25519 . decodeint ( m ) ) )
# C = master _ svk * D
C = ed25519 . scalarmult ( D , ed25519 . decodeint ( master_svk ) )
netbyte = bytearray ( [ 42 if master_address . is_mainnet ( ) else 63 if master_address . is_testnet ( ) else 36 ] )
data = netbyte + ed25519 . encodepoint ( D ) + ed25519 . encodepoint ( C )
checksum = keccak_256 ( data ) . digest ( ) [ : 4 ]
return address . SubAddress ( base58 . encode ( hexlify ( data + checksum ) ) )


signature:
def transfer ( self , address , amount , priority = prio . NORMAL , payment_id = None , unlock_time = 0 , relay = True ) :
    """Sends a transfer from the default account . Returns a list of resulting transactions .
 : param address : destination : class : ` Address < monero . address . Address > ` or subtype
 : param amount : amount to send
 : param priority : transaction priority , implies fee . The priority can be a number
 from 1 to 4 ( unimportant , normal , elevated , priority ) or a constant
 from ` monero . prio ` .
 : param payment _ id : ID for the payment ( must be None if
 : class : ` IntegratedAddress < monero . address . IntegratedAddress > `
 is used as the destination )
 : param unlock _ time : the extra unlock delay
 : param relay : if ` True ` , the wallet will relay the transaction ( s ) to the network
 immediately ; when ` False ` , it will only return the transaction ( s )
 so they might be broadcasted later
 : rtype : list of : class : ` Transaction < monero . transaction . Transaction > `"""

implementation: return self . accounts [ 0 ] . transfer ( address , amount , priority = priority , payment_id = payment_id , unlock_time = unlock_time , relay = relay )


signature:
def transfer_multiple ( self , destinations , priority = prio . NORMAL , payment_id = None , unlock_time = 0 , relay = True ) :
    """Sends a batch of transfers from the default account . Returns a list of resulting
 transactions .
 : param destinations : a list of destination and amount pairs : [ ( address , amount ) , . . . ]
 : param priority : transaction priority , implies fee . The priority can be a number
 from 1 to 4 ( unimportant , normal , elevated , priority ) or a constant
 from ` monero . prio ` .
 : param payment _ id : ID for the payment ( must be None if
 : class : ` IntegratedAddress < monero . address . IntegratedAddress > `
 is used as a destination )
 : param unlock _ time : the extra unlock delay
 : param relay : if ` True ` , the wallet will relay the transaction ( s ) to the network
 immediately ; when ` False ` , it will only return the transaction ( s )
 so they might be broadcasted later
 : rtype : list of : class : ` Transaction < monero . transaction . Transaction > `"""

implementation: return self . accounts [ 0 ] . transfer_multiple ( destinations , priority = priority , payment_id = payment_id , unlock_time = unlock_time , relay = relay )


signature:
def balance ( self , unlocked = False ) :
    """Returns specified balance .
 : param unlocked : if ` True ` , return the unlocked balance , otherwise return total balance
 : rtype : Decimal"""

implementation: return self . _backend . balances ( account = self . index ) [ 1 if unlocked else 0 ]


signature:
def new_address ( self , label = None ) :
    """Creates a new address .
 : param label : address label as ` str `
 : rtype : : class : ` SubAddress < monero . address . SubAddress > `"""

implementation: return self . _backend . new_address ( account = self . index , label = label )


signature:
def transfer ( self , address , amount , priority = prio . NORMAL , payment_id = None , unlock_time = 0 , relay = True ) :
    """Sends a transfer . Returns a list of resulting transactions .
 : param address : destination : class : ` Address < monero . address . Address > ` or subtype
 : param amount : amount to send
 : param priority : transaction priority , implies fee . The priority can be a number
 from 1 to 4 ( unimportant , normal , elevated , priority ) or a constant
 from ` monero . prio ` .
 : param payment _ id : ID for the payment ( must be None if
 : class : ` IntegratedAddress < monero . address . IntegratedAddress > `
 is used as the destination )
 : param unlock _ time : the extra unlock delay
 : param relay : if ` True ` , the wallet will relay the transaction ( s ) to the network
 immediately ; when ` False ` , it will only return the transaction ( s )
 so they might be broadcasted later
 : rtype : list of : class : ` Transaction < monero . transaction . Transaction > `"""

implementation: return self . _backend . transfer ( [ ( address , amount ) ] , priority , payment_id , unlock_time , account = self . index , relay = relay )


signature:
def transfer_multiple ( self , destinations , priority = prio . NORMAL , payment_id = None , unlock_time = 0 , relay = True ) :
    """Sends a batch of transfers . Returns a list of resulting transactions .
 : param destinations : a list of destination and amount pairs :
 [ ( : class : ` Address < monero . address . Address > ` , ` Decimal ` ) , . . . ]
 : param priority : transaction priority , implies fee . The priority can be a number
 from 1 to 4 ( unimportant , normal , elevated , priority ) or a constant
 from ` monero . prio ` .
 : param payment _ id : ID for the payment ( must be None if
 : class : ` IntegratedAddress < monero . address . IntegratedAddress > `
 is used as the destination )
 : param unlock _ time : the extra unlock delay
 : param relay : if ` True ` , the wallet will relay the transaction ( s ) to the network
 immediately ; when ` False ` , it will only return the transaction ( s )
 so they might be broadcasted later
 : rtype : list of : class : ` Transaction < monero . transaction . Transaction > `"""

implementation: return self . _backend . transfer ( destinations , priority , payment_id , unlock_time , account = self . index , relay = relay )


signature:
def to_atomic ( amount ) :
    """Convert Monero decimal to atomic integer of piconero ."""

implementation:
if not isinstance ( amount , ( Decimal , float ) + _integer_types ) :
    raise ValueError ( "Amount '{}' doesn't have numeric type. Only Decimal, int, long and " "float (not recommended) are accepted as amounts." )
return int ( amount * 10 ** 12 )


signature:
def _validate_checksum ( self ) :
    """Given a mnemonic word string , confirm seed checksum ( last word ) matches the computed checksum .
 : rtype : bool"""

implementation:
phrase = self . phrase . split ( " " )
if self . word_list . get_checksum ( self . phrase ) == phrase [ - 1 ] :
    return True
raise ValueError ( "Invalid checksum" )


signature:
def public_address ( self , net = 'mainnet' ) :
    """Returns the master : class : ` Address < monero . address . Address > ` represented by the seed .
 : param net : the network , one of ' mainnet ' , ' testnet ' , ' stagenet ' . Default is ' mainnet ' .
 : rtype : : class : ` Address < monero . address . Address > `"""

implementation:
if net not in ( 'mainnet' , 'testnet' , 'stagenet' ) :
    raise ValueError ( "Invalid net argument. Must be one of ('mainnet', 'testnet', 'stagenet')." )
netbyte = 18 if net == 'mainnet' else 53 if net == 'testnet' else 24
data = "{:x}{:s}{:s}" . format ( netbyte , self . public_spend_key ( ) , self . public_view_key ( ) )
h = keccak_256 ( )
h . update ( unhexlify ( data ) )
checksum = h . hexdigest ( )
return address ( base58 . encode ( data + checksum [ 0 : 8 ] ) )


signature:
def address ( addr , label = None ) :
    """Discover the proper class and return instance for a given Monero address .
 : param addr : the address as a string - like object
 : param label : a label for the address ( defaults to ` None ` )
 : rtype : : class : ` Address ` , : class : ` SubAddress ` or : class : ` IntegratedAddress `"""

implementation:
addr = str ( addr )
if _ADDR_REGEX . match ( addr ) :
    netbyte = bytearray ( unhexlify ( base58 . decode ( addr ) ) ) [ 0 ]
    if netbyte in Address . _valid_netbytes :
        return Address ( addr , label = label )
    elif netbyte in SubAddress . _valid_netbytes :
        return SubAddress ( addr , label = label )
    raise ValueError ( "Invalid address netbyte {nb:x}. Allowed values are: {allowed}" . format ( nb = netbyte , allowed = ", " . join ( map ( lambda b : '%02x' % b , sorted ( Address . _valid_netbytes + SubAddress . _valid_netbytes ) ) ) ) )
elif _IADDR_REGEX . match ( addr ) :
    return IntegratedAddress ( addr )
raise ValueError ( "Address must be either 95 or 106 characters long base58-encoded string, " "is {addr} ({len} chars length)" . format ( addr = addr , len = len ( addr ) ) )


signature:
def with_payment_id ( self , payment_id = 0 ) :
    """Integrates payment id into the address .
 : param payment _ id : int , hexadecimal string or : class : ` PaymentID < monero . numbers . PaymentID > `
 ( max 64 - bit long )
 : rtype : ` IntegratedAddress `
 : raises : ` TypeError ` if the payment id is too long"""

implementation:
payment_id = numbers . PaymentID ( payment_id )
if not payment_id . is_short ( ) :
    raise TypeError ( "Payment ID {0} has more than 64 bits and cannot be integrated" . format ( payment_id ) )
prefix = 54 if self . is_testnet ( ) else 25 if self . is_stagenet ( ) else 19
data = bytearray ( [ prefix ] ) + self . _decoded [ 1 : 65 ] + struct . pack ( '>Q' , int ( payment_id ) )
checksum = bytearray ( keccak_256 ( data ) . digest ( ) [ : 4 ] )
return IntegratedAddress ( base58 . encode ( hexlify ( data + checksum ) ) )


signature:
def base_address ( self ) :
    """Returns the base address without payment id .
 : rtype : : class : ` Address `"""

implementation:
prefix = 53 if self . is_testnet ( ) else 24 if self . is_stagenet ( ) else 18
data = bytearray ( [ prefix ] ) + self . _decoded [ 1 : 65 ]
checksum = keccak_256 ( data ) . digest ( ) [ : 4 ]
return Address ( base58 . encode ( hexlify ( data + checksum ) ) )


signature:
def encode ( hex ) :
    '''Encode hexadecimal string as base58 ( ex : encoding a Monero address ) .'''

implementation:
data = _hexToBin ( hex )
l_data = len ( data )
if l_data == 0 :
    return ""
full_block_count = l_data // __fullBlockSize
last_block_size = l_data % __fullBlockSize
res_size = full_block_count * __fullEncodedBlockSize + __encodedBlockSizes [ last_block_size ]
res = bytearray ( [ __alphabet [ 0 ] ] * res_size )
for i in range ( full_block_count ) :
    res = encode_block ( data [ ( i * __fullBlockSize ) : ( i * __fullBlockSize + __fullBlockSize ) ] , res , i * __fullEncodedBlockSize )
if last_block_size > 0 :
    res = encode_block ( data [ ( full_block_count * __fullBlockSize ) : ( full_block_count * __fullBlockSize + last_block_size ) ] , res , full_block_count * __fullEncodedBlockSize )
return bytes ( res ) . decode ( 'ascii' )


signature:
def decode ( enc ) :
    '''Decode a base58 string ( ex : a Monero address ) into hexidecimal form .'''

implementation:
enc = bytearray ( enc , encoding = 'ascii' )
l_enc = len ( enc )
if l_enc == 0 :
    return ""
full_block_count = l_enc // __fullEncodedBlockSize
last_block_size = l_enc % __fullEncodedBlockSize
try :
    last_block_decoded_size = __encodedBlockSizes . index ( last_block_size )
except ValueError :
    raise ValueError ( "Invalid encoded length: %d" % l_enc )
data_size = full_block_count * __fullBlockSize + last_block_decoded_size
data = bytearray ( data_size )
for i in range ( full_block_count ) :
    data = decode_block ( enc [ ( i * __fullEncodedBlockSize ) : ( i * __fullEncodedBlockSize + __fullEncodedBlockSize ) ] , data , i * __fullBlockSize )
if last_block_size > 0 :
    data = decode_block ( enc [ ( full_block_count * __fullEncodedBlockSize ) : ( full_block_count * __fullEncodedBlockSize + last_block_size ) ] , data , full_block_count * __fullBlockSize )
return _binToHex ( data )


signature:
def encode ( cls , hex ) :
    """Convert hexadecimal string to mnemonic word representation with checksum ."""

implementation:
out = [ ]
for i in range ( len ( hex ) // 8 ) :
    word = endian_swap ( hex [ 8 * i : 8 * i + 8 ] )
    x = int ( word , 16 )
    w1 = x % cls . n
    w2 = ( x // cls . n + w1 ) % cls . n
    w3 = ( x // cls . n // cls . n + w2 ) % cls . n
    out += [ cls . word_list [ w1 ] , cls . word_list [ w2 ] , cls . word_list [ w3 ] ]
checksum = cls . get_checksum ( " " . join ( out ) )
out . append ( checksum )
return " " . join ( out )


signature:
def decode ( cls , phrase ) :
    """Calculate hexadecimal representation of the phrase ."""

implementation:
phrase = phrase . split ( " " )
out = ""
for i in range ( len ( phrase ) // 3 ) :
    word1 , word2 , word3 = phrase [ 3 * i : 3 * i + 3 ]
    w1 = cls . word_list . index ( word1 )
    w2 = cls . word_list . index ( word2 ) % cls . n
    w3 = cls . word_list . index ( word3 ) % cls . n
    x = w1 + cls . n * ( ( w2 - w1 ) % cls . n ) + cls . n * cls . n * ( ( w3 - w2 ) % cls . n )
    out += endian_swap ( "%08x" % x )
return out


signature:
def get_checksum ( cls , phrase ) :
    """Given a mnemonic word string , return a string of the computed checksum .
 : rtype : str"""

implementation:
phrase_split = phrase . split ( " " )
if len ( phrase_split ) < 12 :
    raise ValueError ( "Invalid mnemonic phrase" )
if len ( phrase_split ) > 13 : # Standard format
    phrase = phrase_split [ : 24 ]
else : # MyMonero format
    phrase = phrase_split [ : 12 ]
wstr = "" . join ( word [ : cls . unique_prefix_length ] for word in phrase )
wstr = bytearray ( wstr . encode ( 'utf-8' ) )
z = ( ( crc32 ( wstr ) & 0xffffffff ) ^ 0xffffffff ) >> 0
z2 = ( ( z ^ 0xffffffff ) >> 0 ) % len ( phrase )
return phrase_split [ z2 ]


signature:
def send_transaction ( self , tx , relay = True ) :
    """Sends a transaction generated by a : class : ` Wallet < monero . wallet . Wallet > ` .
 : param tx : : class : ` Transaction < monero . transaction . Transaction > `
 : param relay : whether to relay the transaction to peers . If ` False ` , the daemon will have
 to mine the transaction itself in order to have it included in the blockchain ."""

implementation: return self . _backend . send_transaction ( tx . blob , relay = relay )


signature:
def one ( prompt , * args , ** kwargs ) :
    """Instantiates a picker , registers custom handlers for going back ,
 and starts the picker ."""

implementation:
indicator = '‣'
if sys . version_info < ( 3 , 0 ) :
    indicator = '>'
def go_back ( picker ) :
    return None , - 1
options , verbose_options = prepare_options ( args )
idx = kwargs . get ( 'idx' , 0 )
picker = Picker ( verbose_options , title = prompt , indicator = indicator , default_index = idx )
picker . register_custom_handler ( ord ( 'h' ) , go_back )
picker . register_custom_handler ( curses . KEY_LEFT , go_back )
with stdout_redirected ( sys . stderr ) :
    option , index = picker . start ( )
    if index == - 1 :
        raise QuestionnaireGoBack
    if kwargs . get ( 'return_index' , False ) : # ` one ` was called by a special client , e . g . ` many `
        return index
    return options [ index ]